Backend half
This commit is contained in:
+4
@@ -0,0 +1,4 @@
|
||||
import S3Transport from "./s3-transport";
|
||||
export { default as S3Transport } from "./s3-transport";
|
||||
export * as IS3Transport from "./s3-transport.interface";
|
||||
export default S3Transport;
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.IS3Transport = exports.S3Transport = void 0;
|
||||
const s3_transport_1 = __importDefault(require("./s3-transport"));
|
||||
var s3_transport_2 = require("./s3-transport");
|
||||
Object.defineProperty(exports, "S3Transport", { enumerable: true, get: function () { return __importDefault(s3_transport_2).default; } });
|
||||
exports.IS3Transport = __importStar(require("./s3-transport.interface"));
|
||||
exports.default = s3_transport_1.default;
|
||||
Generated
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
import TransportStream from "winston-transport";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { Options, Config, StreamInfo } from "./s3-transport.interface";
|
||||
declare class S3Transport extends TransportStream {
|
||||
s3Client: S3Client;
|
||||
s3TransportConfig: Required<Config>;
|
||||
streamInfos: Map<string, StreamInfo>;
|
||||
constructor(options: Options);
|
||||
log(log: any, next: () => void): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
export default S3Transport;
|
||||
Generated
Vendored
+89
@@ -0,0 +1,89 @@
|
||||
/// <reference types="node" />
|
||||
import Transport from "winston-transport";
|
||||
import { CompleteMultipartUploadCommandOutput, S3ClientConfig } from "@aws-sdk/client-s3";
|
||||
import { PassThrough } from "stream";
|
||||
/**
|
||||
* Config
|
||||
*/
|
||||
export interface Config<T = any> {
|
||||
/**
|
||||
* bucket
|
||||
*/
|
||||
bucket: string;
|
||||
/**
|
||||
* generateGruop
|
||||
*/
|
||||
generateGruop?: (log: T) => string;
|
||||
/**
|
||||
* generateBucketPath
|
||||
*/
|
||||
generateBucketPath?: (group: string, log: T) => string;
|
||||
/**
|
||||
* maxBufferSize
|
||||
*/
|
||||
maxBufferSize?: number;
|
||||
/**
|
||||
* maxBufferCount
|
||||
*/
|
||||
maxBufferCount?: number;
|
||||
/**
|
||||
* maxFileSize
|
||||
*/
|
||||
maxFileSize?: number;
|
||||
/**
|
||||
* maxFileAge
|
||||
*/
|
||||
maxFileAge?: number;
|
||||
/**
|
||||
* gzip
|
||||
*/
|
||||
gzip?: boolean;
|
||||
}
|
||||
/**
|
||||
* Options
|
||||
*/
|
||||
export interface Options extends Transport.TransportStreamOptions {
|
||||
/**
|
||||
* s3ClientConfig
|
||||
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html
|
||||
*/
|
||||
s3ClientConfig: S3ClientConfig;
|
||||
/**
|
||||
* S3TransportConfig
|
||||
*/
|
||||
s3TransportConfig: Config;
|
||||
}
|
||||
/**
|
||||
* StreamInfoName
|
||||
*/
|
||||
export declare enum StreamInfoName {
|
||||
/**
|
||||
* TotalWrittenBytes
|
||||
*/
|
||||
TotalWrittenBytes = 0,
|
||||
/**
|
||||
* Stream
|
||||
*/
|
||||
Stream = 1,
|
||||
/**
|
||||
* S3Upload
|
||||
*/
|
||||
S3Upload = 2
|
||||
}
|
||||
/**
|
||||
* StreamInfo
|
||||
*/
|
||||
export type StreamInfo = [
|
||||
/**
|
||||
* totalWrittenBytes
|
||||
*/
|
||||
totalWrittenBytes: number,
|
||||
/**
|
||||
* stream
|
||||
*/
|
||||
stream: PassThrough,
|
||||
/**
|
||||
* s3Upload
|
||||
*/
|
||||
s3Upload: Promise<CompleteMultipartUploadCommandOutput>
|
||||
];
|
||||
Generated
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.StreamInfoName = void 0;
|
||||
/**
|
||||
* StreamInfoName
|
||||
*/
|
||||
var StreamInfoName;
|
||||
(function (StreamInfoName) {
|
||||
/**
|
||||
* TotalWrittenBytes
|
||||
*/
|
||||
StreamInfoName[StreamInfoName["TotalWrittenBytes"] = 0] = "TotalWrittenBytes";
|
||||
/**
|
||||
* Stream
|
||||
*/
|
||||
StreamInfoName[StreamInfoName["Stream"] = 1] = "Stream";
|
||||
/**
|
||||
* S3Upload
|
||||
*/
|
||||
StreamInfoName[StreamInfoName["S3Upload"] = 2] = "S3Upload";
|
||||
})(StreamInfoName = exports.StreamInfoName || (exports.StreamInfoName = {}));
|
||||
Generated
Vendored
+182
@@ -0,0 +1,182 @@
|
||||
"use strict";
|
||||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
||||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
||||
return new (P || (P = Promise))(function (resolve, reject) {
|
||||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
||||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
||||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
||||
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
||||
});
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const winston_transport_1 = __importDefault(require("winston-transport"));
|
||||
const client_s3_1 = require("@aws-sdk/client-s3");
|
||||
const lib_storage_1 = require("@aws-sdk/lib-storage");
|
||||
const node_gzip_1 = require("node-gzip");
|
||||
const stream_1 = require("stream");
|
||||
const s3_transport_interface_1 = require("./s3-transport.interface");
|
||||
class S3Transport extends winston_transport_1.default {
|
||||
constructor(options) {
|
||||
super(options);
|
||||
this.streamInfos = new Map();
|
||||
/**
|
||||
* options
|
||||
*/
|
||||
const { s3ClientConfig, s3TransportConfig } = options;
|
||||
/**
|
||||
* default config values
|
||||
*/
|
||||
this.s3TransportConfig = Object.assign({
|
||||
/**
|
||||
* generateGruop
|
||||
*/
|
||||
generateGruop: () => "default",
|
||||
/**
|
||||
* generateBucketPath
|
||||
*/
|
||||
generateBucketPath: () => "",
|
||||
/**
|
||||
* maxBufferCount
|
||||
*/
|
||||
maxBufferCount: 50,
|
||||
/**
|
||||
* maxBufferSize
|
||||
*/
|
||||
maxBufferSize: 1024,
|
||||
/**
|
||||
* maxFileSize
|
||||
*/
|
||||
maxFileSize: 1024 * 2,
|
||||
/**
|
||||
* maxFileAge
|
||||
*/
|
||||
maxFileAge: 1000 * 60 * 5,
|
||||
/**
|
||||
* gzip
|
||||
*/
|
||||
gzip: false }, s3TransportConfig);
|
||||
this.s3Client = new client_s3_1.S3Client(s3ClientConfig);
|
||||
process
|
||||
.on("SIGINT", () => __awaiter(this, void 0, void 0, function* () {
|
||||
yield this.close();
|
||||
process.exit(0);
|
||||
}))
|
||||
.on("beforeExit", () => __awaiter(this, void 0, void 0, function* () {
|
||||
yield this.close();
|
||||
}));
|
||||
}
|
||||
log(log, next) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const { bucket, generateGruop, generateBucketPath, maxBufferSize, maxBufferCount, maxFileSize, maxFileAge, gzip, } = this.s3TransportConfig;
|
||||
/**
|
||||
* Generate the log group for the path.
|
||||
*/
|
||||
const group = generateGruop(log);
|
||||
const data = `${JSON.stringify(log)}\n`;
|
||||
const dataBuffer = gzip ? yield (0, node_gzip_1.gzip)(data) : Buffer.from(data);
|
||||
/**
|
||||
* Get the streamInfo object for the group.
|
||||
*/
|
||||
let groupStreamInfo = this.streamInfos.get(group);
|
||||
/**
|
||||
* If the buffer size exceeds the maximum buffer size, the buffer is flushed.
|
||||
*/
|
||||
if (groupStreamInfo &&
|
||||
groupStreamInfo[s3_transport_interface_1.StreamInfoName.TotalWrittenBytes] +
|
||||
dataBuffer.byteLength >=
|
||||
maxFileSize) {
|
||||
yield new Promise((resolve) => {
|
||||
groupStreamInfo === null || groupStreamInfo === void 0 ? void 0 : groupStreamInfo[s3_transport_interface_1.StreamInfoName.Stream].end(() => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Get the streamInfo object for the group.
|
||||
*/
|
||||
groupStreamInfo = this.streamInfos.get(group);
|
||||
if (groupStreamInfo === undefined) {
|
||||
/**
|
||||
* If the number of buffer size exceeds the maximum number of buffer sizes,
|
||||
* the stream with the most written data is removed and a new stream is created.
|
||||
*/
|
||||
if (this.streamInfos.size >= maxBufferCount) {
|
||||
const sortedStreamInfos = Array.from(this.streamInfos.entries()).sort((a, b) => b[1][0] - a[1][0]);
|
||||
const [, firstStreamInfo] = sortedStreamInfos[0];
|
||||
const [, firstStream] = firstStreamInfo;
|
||||
firstStream.end();
|
||||
}
|
||||
/**
|
||||
* Create a new streamInfo
|
||||
*/
|
||||
const bucketPathStream = new stream_1.PassThrough({
|
||||
highWaterMark: maxBufferSize,
|
||||
});
|
||||
/**
|
||||
* If all data in the buffer is uploaded, the stream is closed and deleted from the streams object
|
||||
*/
|
||||
const uploadPromise = new lib_storage_1.Upload({
|
||||
client: this.s3Client,
|
||||
params: {
|
||||
Bucket: bucket,
|
||||
Key: generateBucketPath(group, log),
|
||||
Body: bucketPathStream,
|
||||
},
|
||||
}).done();
|
||||
/**
|
||||
* If the maximum file age is set,
|
||||
* the stream is automatically closed after the set time.
|
||||
*/
|
||||
let autoFlushProcId;
|
||||
if (maxFileAge > 0) {
|
||||
autoFlushProcId = setTimeout(() => {
|
||||
bucketPathStream.end();
|
||||
}, maxFileAge);
|
||||
}
|
||||
uploadPromise
|
||||
.then(() => {
|
||||
this.streamInfos.delete(group);
|
||||
clearTimeout(autoFlushProcId);
|
||||
})
|
||||
.catch((error) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(error);
|
||||
this.streamInfos.delete(group);
|
||||
clearTimeout(autoFlushProcId);
|
||||
});
|
||||
bucketPathStream.once("error", () => {
|
||||
/**
|
||||
* If an error occurs, delete the stream.
|
||||
*/
|
||||
this.streamInfos.delete(group);
|
||||
clearTimeout(autoFlushProcId);
|
||||
});
|
||||
groupStreamInfo = [0, bucketPathStream, uploadPromise];
|
||||
this.streamInfos.set(group, groupStreamInfo);
|
||||
}
|
||||
/**
|
||||
* Write log data to the stream.
|
||||
*/
|
||||
groupStreamInfo[s3_transport_interface_1.StreamInfoName.Stream].write(dataBuffer);
|
||||
groupStreamInfo[s3_transport_interface_1.StreamInfoName.TotalWrittenBytes] += dataBuffer.length;
|
||||
next === null || next === void 0 ? void 0 : next();
|
||||
});
|
||||
}
|
||||
close() {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
/**
|
||||
* Close streams.
|
||||
*/
|
||||
const pormiseList = [...this.streamInfos.values()].map((groupStreamInfo) => {
|
||||
const [, stream, uploadPromise] = groupStreamInfo;
|
||||
stream.end();
|
||||
return uploadPromise;
|
||||
});
|
||||
yield Promise.all(pormiseList);
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.default = S3Transport;
|
||||
Reference in New Issue
Block a user