Backend half

This commit is contained in:
2025-07-11 19:56:28 +02:00
parent fa868e7c1d
commit 8600fa7c1d
19426 changed files with 3750448 additions and 8108 deletions
@@ -0,0 +1,6 @@
import S3Transport from "./s3-transport";
export { default as S3StreamTransport } from "./stream/s3-transport";
export * as IS3StreamTransport from "./stream/s3-transport.interface";
export { default as S3Transport } from "./s3-transport";
export * as IS3Transport from "./s3-transport.interface";
export default S3Transport;
+37
View File
@@ -0,0 +1,37 @@
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.IS3Transport = exports.S3Transport = exports.IS3StreamTransport = exports.S3StreamTransport = void 0;
const s3_transport_1 = __importDefault(require("./s3-transport"));
var s3_transport_2 = require("./stream/s3-transport");
Object.defineProperty(exports, "S3StreamTransport", { enumerable: true, get: function () { return __importDefault(s3_transport_2).default; } });
exports.IS3StreamTransport = __importStar(require("./stream/s3-transport.interface"));
var s3_transport_3 = require("./s3-transport");
Object.defineProperty(exports, "S3Transport", { enumerable: true, get: function () { return __importDefault(s3_transport_3).default; } });
exports.IS3Transport = __importStar(require("./s3-transport.interface"));
exports.default = s3_transport_1.default;
@@ -0,0 +1,25 @@
import Transport from "winston-transport";
import { S3Client } from "@aws-sdk/client-s3";
import { LogGroup, Options, Config } from "./s3-transport.interface";
declare class S3Transport extends Transport {
logGroups: Record<string, LogGroup>;
s3Client: S3Client;
s3TransportConfig: Required<Config>;
constructor(options: Options);
updateLogGroupList(logGroupIdList?: string[]): Promise<boolean>;
/**
* Create bucket path
*/
static createBucketPath(groupId: string, s3TransportConfig: Config): string;
/**
* Create log group
*/
static createLogGroup(groupId: string, s3TransportConfig: Config): LogGroup;
log(logInfo: any, callback: any): Promise<void>;
static calcDataSize(data: any): number;
/**
* Upload data to s3 bucket
*/
static uploadToS3Bucket(s3Client: S3Client, bucket: string, bucketPath: string, data: any[], compress?: boolean): Promise<boolean>;
}
export default S3Transport;
@@ -0,0 +1,26 @@
import Transport from "winston-transport";
import { S3ClientConfig } from "@aws-sdk/client-s3";
export interface LogGroup {
data: any[];
bucket: string;
bucketPath: string;
createdTime: Date;
isUpdated: boolean;
uploadTime?: Date;
}
export interface Config {
bucket: string;
bucketPath: ((groupId: string) => string) | string;
group?: (<T = any>(logInfo: T) => string) | string;
dataUploadInterval?: number;
maxDataSize?: number;
fileRotationInterval?: number;
gzip?: boolean;
}
export interface Options extends Transport.TransportStreamOptions {
/**
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html
*/
s3ClientConfig: S3ClientConfig;
s3TransportConfig: Config;
}
@@ -0,0 +1,2 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
@@ -0,0 +1,189 @@
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const winston_transport_1 = __importDefault(require("winston-transport"));
const client_s3_1 = require("@aws-sdk/client-s3");
const date_fns_1 = require("date-fns");
const node_gzip_1 = require("node-gzip");
class S3Transport extends winston_transport_1.default {
constructor(options) {
super(options);
/**
* Default config values
*/
this.s3TransportConfig = Object.assign({
/**
* Group for logs classification.
*/
group: "default",
/**
* 1000ms x 20 ⇛ 20s
*/
dataUploadInterval: 1000 * 20,
/**
* 1000ms x 60 ⇛ 60s
*/
fileRotationInterval: 1000 * 60,
/**
* 1000Byte(1KB) x 1000(1MB) x 2 ⇛ 2MB
*/
maxDataSize: 1000 * 1000 * 2,
/**
* Whether to use gzip compression
*/
gzip: false }, options.s3TransportConfig);
this.s3Client = new client_s3_1.S3Client(options.s3ClientConfig);
this.logGroups = {};
setInterval(() => __awaiter(this, void 0, void 0, function* () {
/**
* Upload files to S3 bucket sequentially (not in parallel).
*/
yield this.updateLogGroupList(Object.keys(this.logGroups));
}), this.s3TransportConfig.dataUploadInterval);
}
updateLogGroupList(logGroupIdList = []) {
return __awaiter(this, void 0, void 0, function* () {
const [groupId, ...rest] = logGroupIdList;
if (groupId === undefined) {
/**
* If the item does not exist
*/
return true;
}
const logGroup = this.logGroups[groupId];
if (logGroup.data.length === 0) {
/**
* If the item to be uploaded does not exist
*/
return true;
}
const { bucket, bucketPath, data, uploadTime, createdTime } = logGroup;
const { dataUploadInterval, fileRotationInterval } = this.s3TransportConfig;
if ((0, date_fns_1.isAfter)(new Date(), (0, date_fns_1.addMilliseconds)(createdTime, fileRotationInterval))) {
/**
* Upload data and remove from `logGroups` when file rotation cycle is exceeded
*/
const logData = [...data];
delete this.logGroups[groupId];
yield S3Transport.uploadToS3Bucket(this.s3Client, bucket, bucketPath, logData, this.s3TransportConfig.gzip);
logGroup.bucketPath = S3Transport.createBucketPath(groupId, this.s3TransportConfig);
}
if ((0, date_fns_1.isAfter)(new Date(), (0, date_fns_1.addMilliseconds)(uploadTime || 0, dataUploadInterval)) &&
logGroup.isUpdated) {
/**
* If the data upload cycle has been exceeded
*/
logGroup.uploadTime = new Date();
logGroup.isUpdated = false;
yield S3Transport.uploadToS3Bucket(this.s3Client, bucket, bucketPath, data, this.s3TransportConfig.gzip);
}
return this.updateLogGroupList(rest);
});
}
/**
* Create bucket path
*/
static createBucketPath(groupId, s3TransportConfig) {
const { bucketPath: configBucketPath } = s3TransportConfig;
const bucketPath = typeof configBucketPath === "function"
? configBucketPath(groupId)
: configBucketPath;
return bucketPath;
}
/**
* Create log group
*/
static createLogGroup(groupId, s3TransportConfig) {
const { bucket } = s3TransportConfig;
const bucketPath = S3Transport.createBucketPath(groupId, s3TransportConfig);
return {
bucket,
bucketPath,
data: [],
createdTime: new Date(),
isUpdated: false,
};
}
log(logInfo, callback) {
return __awaiter(this, void 0, void 0, function* () {
/**
* Create a new log group if it doesn't exist
*/
const { group } = this.s3TransportConfig;
const logGroupId = typeof group === "function" ? group(logInfo) : group;
const logGroup = this.logGroups[logGroupId] ||
S3Transport.createLogGroup(logGroupId, this.s3TransportConfig);
if (!(logGroupId in this.logGroups)) {
this.logGroups[logGroupId] = logGroup;
}
/**
* If the data size exceeds the maximum size, the file is uploaded immediately.
*/
const { calcDataSize } = S3Transport;
const isExceededMaxFileSize = calcDataSize(logGroup.data) + calcDataSize(logInfo) >=
this.s3TransportConfig.maxDataSize;
if (isExceededMaxFileSize) {
/**
* If there is not enough space to add a new log, remove the existing log data and upload data
*/
const logData = [...logGroup.data];
logGroup.data = [];
logGroup.createdTime = new Date();
logGroup.uploadTime = new Date();
logGroup.isUpdated = false;
/**
* Upload files
*/
S3Transport.uploadToS3Bucket(this.s3Client, logGroup.bucket, logGroup.bucketPath, logData, this.s3TransportConfig.gzip);
/**
* Create a new bucket path ⇛ Can no longer write files to the existing path
*/
logGroup.bucketPath = S3Transport.createBucketPath(logGroupId, this.s3TransportConfig);
}
/**
* Add log
*/
logGroup.data.push(logInfo);
logGroup.isUpdated = true;
if (typeof callback === "function") {
callback();
}
});
}
static calcDataSize(data) {
return JSON.stringify(data).length;
}
/**
* Upload data to s3 bucket
*/
static uploadToS3Bucket(s3Client, bucket, bucketPath, data, compress = false) {
return __awaiter(this, void 0, void 0, function* () {
const bodyData = data.map((logInfo) => JSON.stringify(logInfo)).join("\n");
let body;
if (compress) {
body = yield (0, node_gzip_1.gzip)(bodyData);
}
else {
body = Buffer.from(bodyData);
}
yield s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: bucket,
Key: bucketPath,
Body: body,
}));
return true;
});
}
}
exports.default = S3Transport;
@@ -0,0 +1,4 @@
import S3Transport from "./s3-transport";
export { default as S3Transport } from "./s3-transport";
export * as IS3Transport from "./s3-transport.interface";
export default S3Transport;
@@ -0,0 +1,34 @@
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.IS3Transport = exports.S3Transport = void 0;
const s3_transport_1 = __importDefault(require("./s3-transport"));
var s3_transport_2 = require("./s3-transport");
Object.defineProperty(exports, "S3Transport", { enumerable: true, get: function () { return __importDefault(s3_transport_2).default; } });
exports.IS3Transport = __importStar(require("./s3-transport.interface"));
exports.default = s3_transport_1.default;
@@ -0,0 +1,12 @@
import TransportStream from "winston-transport";
import { S3Client } from "@aws-sdk/client-s3";
import { Options, Config, StreamInfo } from "./s3-transport.interface";
declare class S3Transport extends TransportStream {
s3Client: S3Client;
s3TransportConfig: Required<Config>;
streamInfos: Map<string, StreamInfo>;
constructor(options: Options);
log(log: any, next: () => void): Promise<void>;
close(): Promise<void>;
}
export default S3Transport;
@@ -0,0 +1,89 @@
/// <reference types="node" />
import Transport from "winston-transport";
import { CompleteMultipartUploadCommandOutput, S3ClientConfig } from "@aws-sdk/client-s3";
import { PassThrough } from "stream";
/**
* Config
*/
export interface Config<T = any> {
/**
* bucket
*/
bucket: string;
/**
* generateGruop
*/
generateGruop?: (log: T) => string;
/**
* generateBucketPath
*/
generateBucketPath?: (group: string, log: T) => string;
/**
* maxBufferSize
*/
maxBufferSize?: number;
/**
* maxBufferCount
*/
maxBufferCount?: number;
/**
* maxFileSize
*/
maxFileSize?: number;
/**
* maxFileAge
*/
maxFileAge?: number;
/**
* gzip
*/
gzip?: boolean;
}
/**
* Options
*/
export interface Options extends Transport.TransportStreamOptions {
/**
* s3ClientConfig
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html
*/
s3ClientConfig: S3ClientConfig;
/**
* S3TransportConfig
*/
s3TransportConfig: Config;
}
/**
* StreamInfoName
*/
export declare enum StreamInfoName {
/**
* TotalWrittenBytes
*/
TotalWrittenBytes = 0,
/**
* Stream
*/
Stream = 1,
/**
* S3Upload
*/
S3Upload = 2
}
/**
* StreamInfo
*/
export type StreamInfo = [
/**
* totalWrittenBytes
*/
totalWrittenBytes: number,
/**
* stream
*/
stream: PassThrough,
/**
* s3Upload
*/
s3Upload: Promise<CompleteMultipartUploadCommandOutput>
];
@@ -0,0 +1,21 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.StreamInfoName = void 0;
/**
* StreamInfoName
*/
var StreamInfoName;
(function (StreamInfoName) {
/**
* TotalWrittenBytes
*/
StreamInfoName[StreamInfoName["TotalWrittenBytes"] = 0] = "TotalWrittenBytes";
/**
* Stream
*/
StreamInfoName[StreamInfoName["Stream"] = 1] = "Stream";
/**
* S3Upload
*/
StreamInfoName[StreamInfoName["S3Upload"] = 2] = "S3Upload";
})(StreamInfoName = exports.StreamInfoName || (exports.StreamInfoName = {}));
@@ -0,0 +1,182 @@
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const winston_transport_1 = __importDefault(require("winston-transport"));
const client_s3_1 = require("@aws-sdk/client-s3");
const lib_storage_1 = require("@aws-sdk/lib-storage");
const node_gzip_1 = require("node-gzip");
const stream_1 = require("stream");
const s3_transport_interface_1 = require("./s3-transport.interface");
class S3Transport extends winston_transport_1.default {
constructor(options) {
super(options);
this.streamInfos = new Map();
/**
* options
*/
const { s3ClientConfig, s3TransportConfig } = options;
/**
* default config values
*/
this.s3TransportConfig = Object.assign({
/**
* generateGruop
*/
generateGruop: () => "default",
/**
* generateBucketPath
*/
generateBucketPath: () => "",
/**
* maxBufferCount
*/
maxBufferCount: 50,
/**
* maxBufferSize
*/
maxBufferSize: 1024,
/**
* maxFileSize
*/
maxFileSize: 1024 * 2,
/**
* maxFileAge
*/
maxFileAge: 1000 * 60 * 5,
/**
* gzip
*/
gzip: false }, s3TransportConfig);
this.s3Client = new client_s3_1.S3Client(s3ClientConfig);
process
.on("SIGINT", () => __awaiter(this, void 0, void 0, function* () {
yield this.close();
process.exit(0);
}))
.on("beforeExit", () => __awaiter(this, void 0, void 0, function* () {
yield this.close();
}));
}
log(log, next) {
return __awaiter(this, void 0, void 0, function* () {
const { bucket, generateGruop, generateBucketPath, maxBufferSize, maxBufferCount, maxFileSize, maxFileAge, gzip, } = this.s3TransportConfig;
/**
* Generate the log group for the path.
*/
const group = generateGruop(log);
const data = `${JSON.stringify(log)}\n`;
const dataBuffer = gzip ? yield (0, node_gzip_1.gzip)(data) : Buffer.from(data);
/**
* Get the streamInfo object for the group.
*/
let groupStreamInfo = this.streamInfos.get(group);
/**
* If the buffer size exceeds the maximum buffer size, the buffer is flushed.
*/
if (groupStreamInfo &&
groupStreamInfo[s3_transport_interface_1.StreamInfoName.TotalWrittenBytes] +
dataBuffer.byteLength >=
maxFileSize) {
yield new Promise((resolve) => {
groupStreamInfo === null || groupStreamInfo === void 0 ? void 0 : groupStreamInfo[s3_transport_interface_1.StreamInfoName.Stream].end(() => {
resolve();
});
});
}
/**
* Get the streamInfo object for the group.
*/
groupStreamInfo = this.streamInfos.get(group);
if (groupStreamInfo === undefined) {
/**
* If the number of buffer size exceeds the maximum number of buffer sizes,
* the stream with the most written data is removed and a new stream is created.
*/
if (this.streamInfos.size >= maxBufferCount) {
const sortedStreamInfos = Array.from(this.streamInfos.entries()).sort((a, b) => b[1][0] - a[1][0]);
const [, firstStreamInfo] = sortedStreamInfos[0];
const [, firstStream] = firstStreamInfo;
firstStream.end();
}
/**
* Create a new streamInfo
*/
const bucketPathStream = new stream_1.PassThrough({
highWaterMark: maxBufferSize,
});
/**
* If all data in the buffer is uploaded, the stream is closed and deleted from the streams object
*/
const uploadPromise = new lib_storage_1.Upload({
client: this.s3Client,
params: {
Bucket: bucket,
Key: generateBucketPath(group, log),
Body: bucketPathStream,
},
}).done();
/**
* If the maximum file age is set,
* the stream is automatically closed after the set time.
*/
let autoFlushProcId;
if (maxFileAge > 0) {
autoFlushProcId = setTimeout(() => {
bucketPathStream.end();
}, maxFileAge);
}
uploadPromise
.then(() => {
this.streamInfos.delete(group);
clearTimeout(autoFlushProcId);
})
.catch((error) => {
// eslint-disable-next-line no-console
console.error(error);
this.streamInfos.delete(group);
clearTimeout(autoFlushProcId);
});
bucketPathStream.once("error", () => {
/**
* If an error occurs, delete the stream.
*/
this.streamInfos.delete(group);
clearTimeout(autoFlushProcId);
});
groupStreamInfo = [0, bucketPathStream, uploadPromise];
this.streamInfos.set(group, groupStreamInfo);
}
/**
* Write log data to the stream.
*/
groupStreamInfo[s3_transport_interface_1.StreamInfoName.Stream].write(dataBuffer);
groupStreamInfo[s3_transport_interface_1.StreamInfoName.TotalWrittenBytes] += dataBuffer.length;
next === null || next === void 0 ? void 0 : next();
});
}
close() {
return __awaiter(this, void 0, void 0, function* () {
/**
* Close streams.
*/
const pormiseList = [...this.streamInfos.values()].map((groupStreamInfo) => {
const [, stream, uploadPromise] = groupStreamInfo;
stream.end();
return uploadPromise;
});
yield Promise.all(pormiseList);
});
}
}
exports.default = S3Transport;