Backend half
This commit is contained in:
+6
@@ -0,0 +1,6 @@
|
||||
import S3Transport from "./s3-transport";
|
||||
export { default as S3StreamTransport } from "./stream/s3-transport";
|
||||
export * as IS3StreamTransport from "./stream/s3-transport.interface";
|
||||
export { default as S3Transport } from "./s3-transport";
|
||||
export * as IS3Transport from "./s3-transport.interface";
|
||||
export default S3Transport;
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.IS3Transport = exports.S3Transport = exports.IS3StreamTransport = exports.S3StreamTransport = void 0;
|
||||
const s3_transport_1 = __importDefault(require("./s3-transport"));
|
||||
var s3_transport_2 = require("./stream/s3-transport");
|
||||
Object.defineProperty(exports, "S3StreamTransport", { enumerable: true, get: function () { return __importDefault(s3_transport_2).default; } });
|
||||
exports.IS3StreamTransport = __importStar(require("./stream/s3-transport.interface"));
|
||||
var s3_transport_3 = require("./s3-transport");
|
||||
Object.defineProperty(exports, "S3Transport", { enumerable: true, get: function () { return __importDefault(s3_transport_3).default; } });
|
||||
exports.IS3Transport = __importStar(require("./s3-transport.interface"));
|
||||
exports.default = s3_transport_1.default;
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
import Transport from "winston-transport";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { LogGroup, Options, Config } from "./s3-transport.interface";
|
||||
declare class S3Transport extends Transport {
|
||||
logGroups: Record<string, LogGroup>;
|
||||
s3Client: S3Client;
|
||||
s3TransportConfig: Required<Config>;
|
||||
constructor(options: Options);
|
||||
updateLogGroupList(logGroupIdList?: string[]): Promise<boolean>;
|
||||
/**
|
||||
* Create bucket path
|
||||
*/
|
||||
static createBucketPath(groupId: string, s3TransportConfig: Config): string;
|
||||
/**
|
||||
* Create log group
|
||||
*/
|
||||
static createLogGroup(groupId: string, s3TransportConfig: Config): LogGroup;
|
||||
log(logInfo: any, callback: any): Promise<void>;
|
||||
static calcDataSize(data: any): number;
|
||||
/**
|
||||
* Upload data to s3 bucket
|
||||
*/
|
||||
static uploadToS3Bucket(s3Client: S3Client, bucket: string, bucketPath: string, data: any[], compress?: boolean): Promise<boolean>;
|
||||
}
|
||||
export default S3Transport;
|
||||
Generated
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
import Transport from "winston-transport";
|
||||
import { S3ClientConfig } from "@aws-sdk/client-s3";
|
||||
export interface LogGroup {
|
||||
data: any[];
|
||||
bucket: string;
|
||||
bucketPath: string;
|
||||
createdTime: Date;
|
||||
isUpdated: boolean;
|
||||
uploadTime?: Date;
|
||||
}
|
||||
export interface Config {
|
||||
bucket: string;
|
||||
bucketPath: ((groupId: string) => string) | string;
|
||||
group?: (<T = any>(logInfo: T) => string) | string;
|
||||
dataUploadInterval?: number;
|
||||
maxDataSize?: number;
|
||||
fileRotationInterval?: number;
|
||||
gzip?: boolean;
|
||||
}
|
||||
export interface Options extends Transport.TransportStreamOptions {
|
||||
/**
|
||||
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html
|
||||
*/
|
||||
s3ClientConfig: S3ClientConfig;
|
||||
s3TransportConfig: Config;
|
||||
}
|
||||
Generated
Vendored
+2
@@ -0,0 +1,2 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
"use strict";
|
||||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
||||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
||||
return new (P || (P = Promise))(function (resolve, reject) {
|
||||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
||||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
||||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
||||
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
||||
});
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const winston_transport_1 = __importDefault(require("winston-transport"));
|
||||
const client_s3_1 = require("@aws-sdk/client-s3");
|
||||
const date_fns_1 = require("date-fns");
|
||||
const node_gzip_1 = require("node-gzip");
|
||||
class S3Transport extends winston_transport_1.default {
|
||||
constructor(options) {
|
||||
super(options);
|
||||
/**
|
||||
* Default config values
|
||||
*/
|
||||
this.s3TransportConfig = Object.assign({
|
||||
/**
|
||||
* Group for logs classification.
|
||||
*/
|
||||
group: "default",
|
||||
/**
|
||||
* 1000ms x 20 ⇛ 20s
|
||||
*/
|
||||
dataUploadInterval: 1000 * 20,
|
||||
/**
|
||||
* 1000ms x 60 ⇛ 60s
|
||||
*/
|
||||
fileRotationInterval: 1000 * 60,
|
||||
/**
|
||||
* 1000Byte(1KB) x 1000(1MB) x 2 ⇛ 2MB
|
||||
*/
|
||||
maxDataSize: 1000 * 1000 * 2,
|
||||
/**
|
||||
* Whether to use gzip compression
|
||||
*/
|
||||
gzip: false }, options.s3TransportConfig);
|
||||
this.s3Client = new client_s3_1.S3Client(options.s3ClientConfig);
|
||||
this.logGroups = {};
|
||||
setInterval(() => __awaiter(this, void 0, void 0, function* () {
|
||||
/**
|
||||
* Upload files to S3 bucket sequentially (not in parallel).
|
||||
*/
|
||||
yield this.updateLogGroupList(Object.keys(this.logGroups));
|
||||
}), this.s3TransportConfig.dataUploadInterval);
|
||||
}
|
||||
updateLogGroupList(logGroupIdList = []) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const [groupId, ...rest] = logGroupIdList;
|
||||
if (groupId === undefined) {
|
||||
/**
|
||||
* If the item does not exist
|
||||
*/
|
||||
return true;
|
||||
}
|
||||
const logGroup = this.logGroups[groupId];
|
||||
if (logGroup.data.length === 0) {
|
||||
/**
|
||||
* If the item to be uploaded does not exist
|
||||
*/
|
||||
return true;
|
||||
}
|
||||
const { bucket, bucketPath, data, uploadTime, createdTime } = logGroup;
|
||||
const { dataUploadInterval, fileRotationInterval } = this.s3TransportConfig;
|
||||
if ((0, date_fns_1.isAfter)(new Date(), (0, date_fns_1.addMilliseconds)(createdTime, fileRotationInterval))) {
|
||||
/**
|
||||
* Upload data and remove from `logGroups` when file rotation cycle is exceeded
|
||||
*/
|
||||
const logData = [...data];
|
||||
delete this.logGroups[groupId];
|
||||
yield S3Transport.uploadToS3Bucket(this.s3Client, bucket, bucketPath, logData, this.s3TransportConfig.gzip);
|
||||
logGroup.bucketPath = S3Transport.createBucketPath(groupId, this.s3TransportConfig);
|
||||
}
|
||||
if ((0, date_fns_1.isAfter)(new Date(), (0, date_fns_1.addMilliseconds)(uploadTime || 0, dataUploadInterval)) &&
|
||||
logGroup.isUpdated) {
|
||||
/**
|
||||
* If the data upload cycle has been exceeded
|
||||
*/
|
||||
logGroup.uploadTime = new Date();
|
||||
logGroup.isUpdated = false;
|
||||
yield S3Transport.uploadToS3Bucket(this.s3Client, bucket, bucketPath, data, this.s3TransportConfig.gzip);
|
||||
}
|
||||
return this.updateLogGroupList(rest);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Create bucket path
|
||||
*/
|
||||
static createBucketPath(groupId, s3TransportConfig) {
|
||||
const { bucketPath: configBucketPath } = s3TransportConfig;
|
||||
const bucketPath = typeof configBucketPath === "function"
|
||||
? configBucketPath(groupId)
|
||||
: configBucketPath;
|
||||
return bucketPath;
|
||||
}
|
||||
/**
|
||||
* Create log group
|
||||
*/
|
||||
static createLogGroup(groupId, s3TransportConfig) {
|
||||
const { bucket } = s3TransportConfig;
|
||||
const bucketPath = S3Transport.createBucketPath(groupId, s3TransportConfig);
|
||||
return {
|
||||
bucket,
|
||||
bucketPath,
|
||||
data: [],
|
||||
createdTime: new Date(),
|
||||
isUpdated: false,
|
||||
};
|
||||
}
|
||||
log(logInfo, callback) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
/**
|
||||
* Create a new log group if it doesn't exist
|
||||
*/
|
||||
const { group } = this.s3TransportConfig;
|
||||
const logGroupId = typeof group === "function" ? group(logInfo) : group;
|
||||
const logGroup = this.logGroups[logGroupId] ||
|
||||
S3Transport.createLogGroup(logGroupId, this.s3TransportConfig);
|
||||
if (!(logGroupId in this.logGroups)) {
|
||||
this.logGroups[logGroupId] = logGroup;
|
||||
}
|
||||
/**
|
||||
* If the data size exceeds the maximum size, the file is uploaded immediately.
|
||||
*/
|
||||
const { calcDataSize } = S3Transport;
|
||||
const isExceededMaxFileSize = calcDataSize(logGroup.data) + calcDataSize(logInfo) >=
|
||||
this.s3TransportConfig.maxDataSize;
|
||||
if (isExceededMaxFileSize) {
|
||||
/**
|
||||
* If there is not enough space to add a new log, remove the existing log data and upload data
|
||||
*/
|
||||
const logData = [...logGroup.data];
|
||||
logGroup.data = [];
|
||||
logGroup.createdTime = new Date();
|
||||
logGroup.uploadTime = new Date();
|
||||
logGroup.isUpdated = false;
|
||||
/**
|
||||
* Upload files
|
||||
*/
|
||||
S3Transport.uploadToS3Bucket(this.s3Client, logGroup.bucket, logGroup.bucketPath, logData, this.s3TransportConfig.gzip);
|
||||
/**
|
||||
* Create a new bucket path ⇛ Can no longer write files to the existing path
|
||||
*/
|
||||
logGroup.bucketPath = S3Transport.createBucketPath(logGroupId, this.s3TransportConfig);
|
||||
}
|
||||
/**
|
||||
* Add log
|
||||
*/
|
||||
logGroup.data.push(logInfo);
|
||||
logGroup.isUpdated = true;
|
||||
if (typeof callback === "function") {
|
||||
callback();
|
||||
}
|
||||
});
|
||||
}
|
||||
static calcDataSize(data) {
|
||||
return JSON.stringify(data).length;
|
||||
}
|
||||
/**
|
||||
* Upload data to s3 bucket
|
||||
*/
|
||||
static uploadToS3Bucket(s3Client, bucket, bucketPath, data, compress = false) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const bodyData = data.map((logInfo) => JSON.stringify(logInfo)).join("\n");
|
||||
let body;
|
||||
if (compress) {
|
||||
body = yield (0, node_gzip_1.gzip)(bodyData);
|
||||
}
|
||||
else {
|
||||
body = Buffer.from(bodyData);
|
||||
}
|
||||
yield s3Client.send(new client_s3_1.PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: bucketPath,
|
||||
Body: body,
|
||||
}));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.default = S3Transport;
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
import S3Transport from "./s3-transport";
|
||||
export { default as S3Transport } from "./s3-transport";
|
||||
export * as IS3Transport from "./s3-transport.interface";
|
||||
export default S3Transport;
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.IS3Transport = exports.S3Transport = void 0;
|
||||
const s3_transport_1 = __importDefault(require("./s3-transport"));
|
||||
var s3_transport_2 = require("./s3-transport");
|
||||
Object.defineProperty(exports, "S3Transport", { enumerable: true, get: function () { return __importDefault(s3_transport_2).default; } });
|
||||
exports.IS3Transport = __importStar(require("./s3-transport.interface"));
|
||||
exports.default = s3_transport_1.default;
|
||||
Generated
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
import TransportStream from "winston-transport";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { Options, Config, StreamInfo } from "./s3-transport.interface";
|
||||
declare class S3Transport extends TransportStream {
|
||||
s3Client: S3Client;
|
||||
s3TransportConfig: Required<Config>;
|
||||
streamInfos: Map<string, StreamInfo>;
|
||||
constructor(options: Options);
|
||||
log(log: any, next: () => void): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
export default S3Transport;
|
||||
Generated
Vendored
+89
@@ -0,0 +1,89 @@
|
||||
/// <reference types="node" />
|
||||
import Transport from "winston-transport";
|
||||
import { CompleteMultipartUploadCommandOutput, S3ClientConfig } from "@aws-sdk/client-s3";
|
||||
import { PassThrough } from "stream";
|
||||
/**
|
||||
* Config
|
||||
*/
|
||||
export interface Config<T = any> {
|
||||
/**
|
||||
* bucket
|
||||
*/
|
||||
bucket: string;
|
||||
/**
|
||||
* generateGruop
|
||||
*/
|
||||
generateGruop?: (log: T) => string;
|
||||
/**
|
||||
* generateBucketPath
|
||||
*/
|
||||
generateBucketPath?: (group: string, log: T) => string;
|
||||
/**
|
||||
* maxBufferSize
|
||||
*/
|
||||
maxBufferSize?: number;
|
||||
/**
|
||||
* maxBufferCount
|
||||
*/
|
||||
maxBufferCount?: number;
|
||||
/**
|
||||
* maxFileSize
|
||||
*/
|
||||
maxFileSize?: number;
|
||||
/**
|
||||
* maxFileAge
|
||||
*/
|
||||
maxFileAge?: number;
|
||||
/**
|
||||
* gzip
|
||||
*/
|
||||
gzip?: boolean;
|
||||
}
|
||||
/**
|
||||
* Options
|
||||
*/
|
||||
export interface Options extends Transport.TransportStreamOptions {
|
||||
/**
|
||||
* s3ClientConfig
|
||||
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html
|
||||
*/
|
||||
s3ClientConfig: S3ClientConfig;
|
||||
/**
|
||||
* S3TransportConfig
|
||||
*/
|
||||
s3TransportConfig: Config;
|
||||
}
|
||||
/**
|
||||
* StreamInfoName
|
||||
*/
|
||||
export declare enum StreamInfoName {
|
||||
/**
|
||||
* TotalWrittenBytes
|
||||
*/
|
||||
TotalWrittenBytes = 0,
|
||||
/**
|
||||
* Stream
|
||||
*/
|
||||
Stream = 1,
|
||||
/**
|
||||
* S3Upload
|
||||
*/
|
||||
S3Upload = 2
|
||||
}
|
||||
/**
|
||||
* StreamInfo
|
||||
*/
|
||||
export type StreamInfo = [
|
||||
/**
|
||||
* totalWrittenBytes
|
||||
*/
|
||||
totalWrittenBytes: number,
|
||||
/**
|
||||
* stream
|
||||
*/
|
||||
stream: PassThrough,
|
||||
/**
|
||||
* s3Upload
|
||||
*/
|
||||
s3Upload: Promise<CompleteMultipartUploadCommandOutput>
|
||||
];
|
||||
Generated
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.StreamInfoName = void 0;
|
||||
/**
|
||||
* StreamInfoName
|
||||
*/
|
||||
var StreamInfoName;
|
||||
(function (StreamInfoName) {
|
||||
/**
|
||||
* TotalWrittenBytes
|
||||
*/
|
||||
StreamInfoName[StreamInfoName["TotalWrittenBytes"] = 0] = "TotalWrittenBytes";
|
||||
/**
|
||||
* Stream
|
||||
*/
|
||||
StreamInfoName[StreamInfoName["Stream"] = 1] = "Stream";
|
||||
/**
|
||||
* S3Upload
|
||||
*/
|
||||
StreamInfoName[StreamInfoName["S3Upload"] = 2] = "S3Upload";
|
||||
})(StreamInfoName = exports.StreamInfoName || (exports.StreamInfoName = {}));
|
||||
Generated
Vendored
+182
@@ -0,0 +1,182 @@
|
||||
"use strict";
|
||||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
||||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
||||
return new (P || (P = Promise))(function (resolve, reject) {
|
||||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
||||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
||||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
||||
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
||||
});
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const winston_transport_1 = __importDefault(require("winston-transport"));
|
||||
const client_s3_1 = require("@aws-sdk/client-s3");
|
||||
const lib_storage_1 = require("@aws-sdk/lib-storage");
|
||||
const node_gzip_1 = require("node-gzip");
|
||||
const stream_1 = require("stream");
|
||||
const s3_transport_interface_1 = require("./s3-transport.interface");
|
||||
class S3Transport extends winston_transport_1.default {
|
||||
constructor(options) {
|
||||
super(options);
|
||||
this.streamInfos = new Map();
|
||||
/**
|
||||
* options
|
||||
*/
|
||||
const { s3ClientConfig, s3TransportConfig } = options;
|
||||
/**
|
||||
* default config values
|
||||
*/
|
||||
this.s3TransportConfig = Object.assign({
|
||||
/**
|
||||
* generateGruop
|
||||
*/
|
||||
generateGruop: () => "default",
|
||||
/**
|
||||
* generateBucketPath
|
||||
*/
|
||||
generateBucketPath: () => "",
|
||||
/**
|
||||
* maxBufferCount
|
||||
*/
|
||||
maxBufferCount: 50,
|
||||
/**
|
||||
* maxBufferSize
|
||||
*/
|
||||
maxBufferSize: 1024,
|
||||
/**
|
||||
* maxFileSize
|
||||
*/
|
||||
maxFileSize: 1024 * 2,
|
||||
/**
|
||||
* maxFileAge
|
||||
*/
|
||||
maxFileAge: 1000 * 60 * 5,
|
||||
/**
|
||||
* gzip
|
||||
*/
|
||||
gzip: false }, s3TransportConfig);
|
||||
this.s3Client = new client_s3_1.S3Client(s3ClientConfig);
|
||||
process
|
||||
.on("SIGINT", () => __awaiter(this, void 0, void 0, function* () {
|
||||
yield this.close();
|
||||
process.exit(0);
|
||||
}))
|
||||
.on("beforeExit", () => __awaiter(this, void 0, void 0, function* () {
|
||||
yield this.close();
|
||||
}));
|
||||
}
|
||||
log(log, next) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const { bucket, generateGruop, generateBucketPath, maxBufferSize, maxBufferCount, maxFileSize, maxFileAge, gzip, } = this.s3TransportConfig;
|
||||
/**
|
||||
* Generate the log group for the path.
|
||||
*/
|
||||
const group = generateGruop(log);
|
||||
const data = `${JSON.stringify(log)}\n`;
|
||||
const dataBuffer = gzip ? yield (0, node_gzip_1.gzip)(data) : Buffer.from(data);
|
||||
/**
|
||||
* Get the streamInfo object for the group.
|
||||
*/
|
||||
let groupStreamInfo = this.streamInfos.get(group);
|
||||
/**
|
||||
* If the buffer size exceeds the maximum buffer size, the buffer is flushed.
|
||||
*/
|
||||
if (groupStreamInfo &&
|
||||
groupStreamInfo[s3_transport_interface_1.StreamInfoName.TotalWrittenBytes] +
|
||||
dataBuffer.byteLength >=
|
||||
maxFileSize) {
|
||||
yield new Promise((resolve) => {
|
||||
groupStreamInfo === null || groupStreamInfo === void 0 ? void 0 : groupStreamInfo[s3_transport_interface_1.StreamInfoName.Stream].end(() => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Get the streamInfo object for the group.
|
||||
*/
|
||||
groupStreamInfo = this.streamInfos.get(group);
|
||||
if (groupStreamInfo === undefined) {
|
||||
/**
|
||||
* If the number of buffer size exceeds the maximum number of buffer sizes,
|
||||
* the stream with the most written data is removed and a new stream is created.
|
||||
*/
|
||||
if (this.streamInfos.size >= maxBufferCount) {
|
||||
const sortedStreamInfos = Array.from(this.streamInfos.entries()).sort((a, b) => b[1][0] - a[1][0]);
|
||||
const [, firstStreamInfo] = sortedStreamInfos[0];
|
||||
const [, firstStream] = firstStreamInfo;
|
||||
firstStream.end();
|
||||
}
|
||||
/**
|
||||
* Create a new streamInfo
|
||||
*/
|
||||
const bucketPathStream = new stream_1.PassThrough({
|
||||
highWaterMark: maxBufferSize,
|
||||
});
|
||||
/**
|
||||
* If all data in the buffer is uploaded, the stream is closed and deleted from the streams object
|
||||
*/
|
||||
const uploadPromise = new lib_storage_1.Upload({
|
||||
client: this.s3Client,
|
||||
params: {
|
||||
Bucket: bucket,
|
||||
Key: generateBucketPath(group, log),
|
||||
Body: bucketPathStream,
|
||||
},
|
||||
}).done();
|
||||
/**
|
||||
* If the maximum file age is set,
|
||||
* the stream is automatically closed after the set time.
|
||||
*/
|
||||
let autoFlushProcId;
|
||||
if (maxFileAge > 0) {
|
||||
autoFlushProcId = setTimeout(() => {
|
||||
bucketPathStream.end();
|
||||
}, maxFileAge);
|
||||
}
|
||||
uploadPromise
|
||||
.then(() => {
|
||||
this.streamInfos.delete(group);
|
||||
clearTimeout(autoFlushProcId);
|
||||
})
|
||||
.catch((error) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(error);
|
||||
this.streamInfos.delete(group);
|
||||
clearTimeout(autoFlushProcId);
|
||||
});
|
||||
bucketPathStream.once("error", () => {
|
||||
/**
|
||||
* If an error occurs, delete the stream.
|
||||
*/
|
||||
this.streamInfos.delete(group);
|
||||
clearTimeout(autoFlushProcId);
|
||||
});
|
||||
groupStreamInfo = [0, bucketPathStream, uploadPromise];
|
||||
this.streamInfos.set(group, groupStreamInfo);
|
||||
}
|
||||
/**
|
||||
* Write log data to the stream.
|
||||
*/
|
||||
groupStreamInfo[s3_transport_interface_1.StreamInfoName.Stream].write(dataBuffer);
|
||||
groupStreamInfo[s3_transport_interface_1.StreamInfoName.TotalWrittenBytes] += dataBuffer.length;
|
||||
next === null || next === void 0 ? void 0 : next();
|
||||
});
|
||||
}
|
||||
close() {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
/**
|
||||
* Close streams.
|
||||
*/
|
||||
const pormiseList = [...this.streamInfos.values()].map((groupStreamInfo) => {
|
||||
const [, stream, uploadPromise] = groupStreamInfo;
|
||||
stream.end();
|
||||
return uploadPromise;
|
||||
});
|
||||
yield Promise.all(pormiseList);
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.default = S3Transport;
|
||||
Reference in New Issue
Block a user