mirror of
https://github.com/diced/zipline.git
synced 2025-12-12 15:50:11 -08:00
fix: scripts
This commit is contained in:
@@ -86,4 +86,6 @@ export async function exportConfig({ yml, showDefaults }: { yml?: boolean; showD
|
||||
|
||||
console.log(`${yml ? '- ' : ''}${envVar.variable}=${envValue}`);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { bytes } from '@/lib/bytes';
|
||||
import { config, reloadSettings } from '@/lib/config';
|
||||
import { guess } from '@/lib/mimes';
|
||||
import { statSync } from 'fs';
|
||||
import { readFile, readdir } from 'fs/promises';
|
||||
import { mkdir, readdir } from 'fs/promises';
|
||||
import { join, parse, resolve } from 'path';
|
||||
import { reloadSettings } from '@/lib/config';
|
||||
import { bytes } from '@/lib/bytes';
|
||||
|
||||
export async function importDir(
|
||||
directory: string,
|
||||
@@ -57,16 +57,18 @@ export async function importDir(
|
||||
|
||||
for (let i = 0; i !== files.length; ++i) {
|
||||
const info = parse(files[i]);
|
||||
if (info.base.startsWith('.thumbnail')) continue;
|
||||
|
||||
const mime = await guess(info.ext.replace('.', ''));
|
||||
const { size } = statSync(join(fullPath, files[i]));
|
||||
|
||||
data[i] = {
|
||||
data.push({
|
||||
name: info.base,
|
||||
type: mime,
|
||||
size,
|
||||
userId,
|
||||
...(folder ? { folderId: folder } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
if (!skipDb) {
|
||||
@@ -78,16 +80,25 @@ export async function importDir(
|
||||
|
||||
const totalSize = data.reduce((acc, file) => acc + file.size, 0);
|
||||
let completed = 0;
|
||||
let imported = 0;
|
||||
|
||||
if (config.datasource.type === 'local')
|
||||
await mkdir(config.datasource.local!.directory, { recursive: true });
|
||||
|
||||
const { getDatasource } = await import('@/lib/datasource/index.js');
|
||||
const datasource = getDatasource(config);
|
||||
if (!datasource) return console.error('No datasource configured');
|
||||
|
||||
for (let i = 0; i !== data.length; ++i) {
|
||||
if (!data[i]) continue;
|
||||
|
||||
const { datasource } = await import('@/lib/datasource/index.js');
|
||||
for (let i = 0; i !== files.length; ++i) {
|
||||
console.log(`Uploading ${data[i].name} (${bytes(data[i].size)})...`);
|
||||
|
||||
const start = process.hrtime();
|
||||
|
||||
const buff = await readFile(join(fullPath, files[i]));
|
||||
await datasource.put(data[i].name, buff, {
|
||||
await datasource.put(data[i].name, join(fullPath, files[i]), {
|
||||
mimetype: data[i].type ?? 'application/octet-stream',
|
||||
noDelete: true,
|
||||
});
|
||||
|
||||
const diff = process.hrtime(start);
|
||||
@@ -104,7 +115,11 @@ export async function importDir(
|
||||
console.log(
|
||||
`Uploaded ${data[i].name} in ${timeStr} (${bytes(data[i].size)}) ${i + 1}/${files.length} ${bytes(completed)}/${bytes(totalSize)} ${uploadSpeedStr}`,
|
||||
);
|
||||
|
||||
++imported;
|
||||
}
|
||||
|
||||
console.log('Done importing files.');
|
||||
console.log(`Done importing ${imported} files.`);
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -27,4 +27,5 @@ export async function listUsers({ extra, format, id }: { extra?: string[]; forma
|
||||
});
|
||||
|
||||
console.log(JSON.stringify(users, null, format ? 2 : 0));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -6,4 +6,5 @@ export async function readConfig({ format }: { format: boolean }) {
|
||||
const { config } = await import('@/lib/config/index.js');
|
||||
|
||||
console.log(JSON.stringify(config, null, format ? 2 : 0));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -34,4 +34,5 @@ export async function setUser(property: string, value: string, { id }: { id: str
|
||||
if (property === 'password') parsed = '*********';
|
||||
|
||||
console.log(`updated user(${id}) -> ${property} = ${parsed || value}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export type PutOptions = { mimetype?: string; noDelete?: boolean };
|
||||
|
||||
export abstract class Datasource {
|
||||
public name: string | undefined;
|
||||
|
||||
public abstract get(file: string): null | Readable | Promise<Readable | null>;
|
||||
public abstract put(file: string, data: Buffer | string, options?: { mimetype?: string }): Promise<void>;
|
||||
public abstract put(file: string, data: Buffer | string, options?: PutOptions): Promise<void>;
|
||||
public abstract delete(file: string | string[]): Promise<void>;
|
||||
public abstract size(file: string): Promise<number>;
|
||||
public abstract totalSize(): Promise<number>;
|
||||
|
||||
@@ -2,7 +2,7 @@ import { createReadStream, existsSync } from 'fs';
|
||||
import { access, constants, copyFile, readdir, rename, rm, stat, writeFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { Datasource } from './Datasource';
|
||||
import { Datasource, PutOptions } from './Datasource';
|
||||
|
||||
async function existsAndCanRW(path: string): Promise<boolean> {
|
||||
try {
|
||||
@@ -29,7 +29,7 @@ export class LocalDatasource extends Datasource {
|
||||
return readStream;
|
||||
}
|
||||
|
||||
public async put(file: string, data: Buffer | string): Promise<void> {
|
||||
public async put(file: string, data: Buffer | string, { noDelete }: PutOptions): Promise<void> {
|
||||
const path = join(this.dir, file);
|
||||
|
||||
// handles if given a path to a file, it will just move it instead of doing unecessary writes
|
||||
@@ -41,7 +41,8 @@ export class LocalDatasource extends Datasource {
|
||||
);
|
||||
|
||||
await copyFile(data, path);
|
||||
await rm(data);
|
||||
|
||||
if (!noDelete) await rm(data);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import { Readable } from 'stream';
|
||||
import { ReadableStream } from 'stream/web';
|
||||
import Logger, { log } from '../logger';
|
||||
import { randomCharacters } from '../random';
|
||||
import { Datasource } from './Datasource';
|
||||
import { Datasource, PutOptions } from './Datasource';
|
||||
|
||||
function isOk(code: number) {
|
||||
return code >= 200 && code < 300;
|
||||
@@ -160,13 +160,7 @@ export class S3Datasource extends Datasource {
|
||||
}
|
||||
}
|
||||
|
||||
public async put(
|
||||
file: string,
|
||||
data: Buffer | string,
|
||||
options: {
|
||||
mimetype?: string;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
public async put(file: string, data: Buffer | string, options: PutOptions = {}): Promise<void> {
|
||||
let command = new PutObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: this.key(file),
|
||||
|
||||
@@ -11,7 +11,7 @@ declare global {
|
||||
var __datasource__: Datasource;
|
||||
}
|
||||
|
||||
function getDatasource(config?: Config): void {
|
||||
function getDatasource(config?: Config): Datasource | void {
|
||||
if (!config) return;
|
||||
|
||||
const logger = log('datasource');
|
||||
@@ -35,6 +35,8 @@ function getDatasource(config?: Config): void {
|
||||
logger.error(`Datasource type ${config.datasource.type} is not supported`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return datasource;
|
||||
}
|
||||
|
||||
datasource = global.__datasource__;
|
||||
|
||||
Reference in New Issue
Block a user