feat: Implemented emulator installation

feat: Updated romm API version
feat: Updated es-de rules
feat: Added tabs to game details
refactor: returned to global query definitions to help with typescript performance
This commit is contained in:
Simeon Radivoev 2026-03-22 01:11:21 +02:00
parent cf6fff6fac
commit 3750e9ed8f
Signed by: simeonradivoev
GPG key ID: 7611A451D2A5D37A
103 changed files with 4888 additions and 1632 deletions

View file

@ -0,0 +1,105 @@
import { EmulatorPackageType } from "@/shared/constants";
import { getStoreEmulatorPackage } from "../store/services/gamesService";
import { IJob, JobContext } from "../task-queue";
import z from "zod";
import { Glob } from "bun";
import { config } from "../app";
import path from 'node:path';
import { getOrCachedGithubRelease } from "../cache";
import _7z from '7zip-min';
import fs from "node:fs/promises";
import { Downloader } from "@/bun/utils/downloader";
import { move } from "fs-extra";
type EmulatorDownloadStates = "download" | "extract";
export class EmulatorDownloadJob implements IJob<z.infer<typeof EmulatorDownloadJob.dataSchema>, EmulatorDownloadStates>
{
static id = "download-emulator" as const;
static dataSchema = z.object({ emulator: z.string() });
emulator: string;
downloadSource: string;
emulatorPackage?: EmulatorPackageType;
constructor(emulator: string, downloadSource: string)
{
this.emulator = emulator;
this.downloadSource = downloadSource;
}
async start (context: JobContext<EmulatorDownloadJob, z.infer<typeof EmulatorDownloadJob.dataSchema>, EmulatorDownloadStates>)
{
this.emulatorPackage = await getStoreEmulatorPackage(this.emulator);
if (!this.emulatorPackage) throw new Error("Emulator not found");
if (!this.emulatorPackage.downloads) throw new Error("Emulator has no downloads");
const validDownloads = this.emulatorPackage.downloads[`${process.platform}:${process.arch}`];
if (!validDownloads) throw new Error(`Now downloads in ${this.emulatorPackage.name} for platform ${process.platform}:${process.arch}`);
const validDownload = validDownloads.find(d => d.type === this.downloadSource);
if (!validDownload || !validDownload.path) throw new Error(`Download type ${this.downloadSource} not found`);
console.log("Trying To Download from ", `https://api.github.com/repos/${validDownload.path}/releases/latest`);
const latestRelease = await getOrCachedGithubRelease(validDownload.path);
const glob = new Glob(validDownload.pattern);
const validAsset = latestRelease.assets.find(a => glob.match(a.name));
if (!validAsset) throw new Error("Could Not Find Valid Asset");
const downloadUrl = validAsset.browser_download_url;
const emulatorsFolder = path.join(config.get('downloadPath'), "emulators", this.emulator);
const isArchive = validAsset.content_type === 'application/x-7z-compressed' || validAsset.name.endsWith('.7z') || validAsset.content_type === 'application/zip' || validAsset.name.endsWith('.zip');
const isAppImage = validAsset.name.endsWith(".AppImage");
if (!isArchive && !isAppImage)
{
throw new Error("Invalid Download Type");
}
const tmpFolder = path.join(config.get("downloadPath"), ".tmp");
const downloader = new Downloader(this.emulator,
[{ url: new URL(downloadUrl), file_name: path.basename(downloadUrl), file_path: this.emulator }],
tmpFolder,
{
onProgress (stats)
{
context.setProgress(stats.progress, 'download');
},
});
const destinationPaths = await downloader.start();
if (destinationPaths)
{
if (isArchive)
{
if (await downloader.start() && destinationPaths[0])
{
let destinationPath = destinationPaths[0];
await _7z.unpack(destinationPath, emulatorsFolder);
await fs.rm(destinationPath, { recursive: true });
// check if 1 root folder we need to get rid of
const contents = await fs.readdir(emulatorsFolder);
if (contents.length === 1)
{
const stat = await fs.stat(path.join(emulatorsFolder, contents[0]));
if (stat.isDirectory())
{
console.log("Found 1 root folder, using that instead");
const tmpEmulatorsFolder = `${emulatorsFolder} (1)`;
await move(path.join(emulatorsFolder, contents[0]), tmpEmulatorsFolder, { overwrite: true });
await move(tmpEmulatorsFolder, emulatorsFolder, { overwrite: true });
}
}
}
}
}
}
exposeData ()
{
return { emulator: this.emulator };
}
}

View file

@ -6,12 +6,14 @@ import * as schema from "@schema/app";
import * as emulatorSchema from "@schema/emulators";
import path from 'node:path';
import { getPlatformApiPlatformsIdGet, getRomApiRomsIdGet, PlatformSchema } from "@clients/romm";
import { config, db, emulatorsDb, jar } from "../app";
import unzip from 'unzip-stream';
import { Readable, Transform } from "node:stream";
import { config, db, emulatorsDb, events, jar } from "../app";
import { extractStoreGameSourceId, getStoreGameFromId } from "../store/services/gamesService";
import * as igdb from 'ts-igdb-client';
import secrets from "../secrets";
import { hashFile } from "@/bun/utils";
import { Downloader } from "@/bun/utils/downloader";
import { sleep } from "bun";
import _7z from '7zip-min';
interface JobConfig
{
@ -19,13 +21,16 @@ interface JobConfig
dryDownload?: boolean;
}
export class InstallJob implements IJob
export type InstallJobStates = 'download' | 'extract';
export class InstallJob implements IJob<never, InstallJobStates>
{
public gameId: string;
public source: string;
public sourceId: string;
public config?: JobConfig;
static id = "install-job" as const;
public group = InstallJob.id;
constructor(id: string, source: string, sourceId: string, config?: JobConfig)
{
@ -35,162 +40,124 @@ export class InstallJob implements IJob
this.source = source;
}
public async start (cx: JobContext)
public async start (cx: JobContext<InstallJob, never, InstallJobStates>)
{
cx.setProgress(0, 'download');
fs.mkdir(config.get('downloadPath'), { recursive: true });
const downloadPath = config.get('downloadPath');
let files: {
url: URL,
file_path: string;
file_name: string;
size?: number;
}[] = [];
let cookie: string = '';
let screenshotUrls: string[];
let coverUrl: string;
let rommPlatform: PlatformSchema | undefined;
let slug: string | null;
let path_fs: string | undefined;
let summary: string | null;
let name: string | null;
let last_played: Date | null;
let igdb_id: number | null;
let ra_id: number | null;
let source_id: string;
let system_slug: string;
let extract_path: string;
let metadata: any | undefined;
switch (this.source)
{
case 'romm':
const rom = (await getRomApiRomsIdGet({ path: { id: Number(this.gameId) }, throwOnError: true })).data;
rommPlatform = (await getPlatformApiPlatformsIdGet({ path: { id: rom.platform_id }, throwOnError: true })).data;
const rommAddress = config.get('rommAddress');
coverUrl = `${rommAddress}${rom.path_cover_large}`;
screenshotUrls = rom.merged_screenshots.map(s => `${config.get('rommAddress')}${s}`);
last_played = rom.rom_user.last_played ? new Date(rom.rom_user.last_played) : null;
igdb_id = rom.igdb_id;
ra_id = rom.ra_id;
summary = rom.summary;
name = rom.name;
path_fs = path.join(rom.fs_path, rom.fs_name);
source_id = String(rom.id);
slug = rom.slug;
system_slug = rommPlatform.slug;
extract_path = '';
metadata = rom.metadatum;
const rommFiles = await Promise.all(rom.files.map(async f =>
{
const localPath = path.join(config.get('downloadPath'), f.full_path);
if (f.md5_hash && await fs.exists(localPath))
{
const existingHash = await hashFile(localPath, 'sha1');
if (existingHash === f.md5_hash)
{
console.log("File Already Present: ", f.full_path);
return undefined;
}
console.warn("File ", f.full_path, 'with hash', existingHash, 'has different hash than', f.sha1_hash);
}
return {
url: new URL(`${config.get('rommAddress')}/api/romsfiles/${f.id}/content/${f.file_name}`),
file_name: f.file_name,
file_path: path.join(config.get('downloadPath'), f.file_path),
size: f.file_size_bytes
};
}));
files.push(...rommFiles.filter(f => f !== undefined));
cookie = await jar.getCookieString(config.get('rommAddress') ?? '');
break;
case 'store':
const game = await getStoreGameFromId(this.gameId);
const gameId = extractStoreGameSourceId(this.gameId);
coverUrl = game.pictures.titlescreens[0];
screenshotUrls = game.pictures.screenshots;
files.push({ url: new URL(game.file), file_path: `roms/${game.system}`, file_name: path.basename(decodeURI(game.file)) });
slug = this.gameId;
source_id = this.gameId;
name = game.title;
summary = game.description;
system_slug = gameId.system;
extract_path = path.join('roms', gameId.system);
break;
default:
throw new Error("Unsupported source");
}
if (this.config?.dryRun !== true)
{
const downloadPath = config.get('downloadPath');
let downloadUrl: URL;
let cookie: string = '';
let screenshotUrls: string[];
let coverUrl: string;
let rommPlatform: PlatformSchema | undefined;
let slug: string | null;
let path_fs: string | undefined;
let summary: string | null;
let name: string | null;
let last_played: Date | null;
let igdb_id: number | null;
let ra_id: number | null;
let source_id: string;
let system_slug: string;
let extract_path: string;
switch (this.source)
{
case 'romm':
const rom = (await getRomApiRomsIdGet({ path: { id: Number(this.gameId) }, throwOnError: true })).data;
rommPlatform = (await getPlatformApiPlatformsIdGet({ path: { id: rom.platform_id }, throwOnError: true })).data;
const rommAddress = config.get('rommAddress');
coverUrl = `${rommAddress}${rom.path_cover_large}`;
screenshotUrls = rom.merged_screenshots.map(s => `${config.get('rommAddress')}${s}`);
last_played = rom.rom_user.last_played ? new Date(rom.rom_user.last_played) : null;
igdb_id = rom.igdb_id;
ra_id = rom.ra_id;
summary = rom.summary;
name = rom.name;
path_fs = path.join(rom.fs_path, rom.fs_name);
source_id = String(rom.id);
slug = rom.slug;
system_slug = rommPlatform.slug;
extract_path = '';
downloadUrl = new URL(`${config.get('rommAddress')}/api/roms/download`);
downloadUrl.searchParams.set('rom_ids', String(this.gameId));
cookie = await jar.getCookieString(config.get('rommAddress') ?? '');
break;
case 'store':
const game = await getStoreGameFromId(this.gameId);
const gameId = extractStoreGameSourceId(this.gameId);
coverUrl = game.pictures.titlescreens[0];
screenshotUrls = game.pictures.screenshots;
downloadUrl = new URL(game.file);
slug = this.gameId;
source_id = this.gameId;
name = game.title;
summary = game.description;
system_slug = gameId.system;
extract_path = 'roms', gameId.system;
break;
default:
throw new Error("Unsupported source");
}
if (this.config?.dryDownload !== true)
{
/*
// download files for rom
const downloadUrl = new URL(`${config.get('rommAddress')}/api/roms/download`);
downloadUrl.searchParams.set('rom_ids', String(this.id));
const downloader = new DownloaderHelper(downloadUrl.href, downloadPath, {
headers: {
cookie: await jar.getCookieString(config.get('rommAddress') ?? '')
},
fileName: `${this.id}.zip`,
// Romm doesn't support resume download
override: true
});
cx.abortSignal.addEventListener('abort', downloader.stop);
downloader.on('progress.throttled', e =>
{
cx.setProgress(e.progress, 'download');
});
downloader.on('error', (e) =>
{
cx.abort(e);
});
const finishPromise = new Promise<string>(resolve =>
{
downloader.on("end", ({ filePath }) => resolve(filePath));
});
await downloader.start().catch(err => console.error(err));
const zipFilePath = await finishPromise;
cx.setProgress(0, 'extract');
const zip = new StreamZip.async({ file: zipFilePath });
const totalCount = await zip.entriesCount;
let extractCount = 0;
zip.on('extract', async (entry, file) =>
{
console.log(`Extracted ${entry.name} to ${file}`);
cx.setProgress(extractCount / totalCount * 100, 'extract');
extractCount++;
});
await zip.extract(null, downloadPath);
await zip.close();
await fs.rm(zipFilePath);*/
cx.setProgress(0, 'download');
const res = await fetch(downloadUrl, {
headers: {
cookie: cookie
},
});
const totalBytes = Number(res.headers.get("content-length")) || 0;
let bytesReceived = 0;
const progressStream = new Transform({
transform (chunk, _, callback)
const downloader = new Downloader(`game-${this.source}-${this.gameId}`,
files,
config.get('downloadPath'),
{
bytesReceived += chunk.length;
if (totalBytes > 0)
signal: cx.abortSignal,
onProgress (stats)
{
const percent = (bytesReceived / totalBytes) * 100;
cx.setProgress(percent, 'download');
}
this.push(chunk);
callback();
}
});
await new Promise((resolve, reject) =>
{
const extract = unzip.Extract({ path: path.join(downloadPath, extract_path), });
(extract as any).unzipStream.on('entry', (entry: any) =>
{
if (!path_fs)
path_fs = path.join(extract_path, entry.path);
cx.setProgress(stats.progress, 'download');
},
});
Readable.fromWeb(res.body as any).pipe(progressStream)
.pipe(extract)
.on('close', resolve)
.on('error', reject);
});
const downloadedFiles = await downloader.start();
if (extract_path && downloadedFiles)
{
for (const path of downloadedFiles)
{
await _7z.unpack(path, extract_path);
}
}
}
if (this.config?.dryDownload === true)
@ -198,8 +165,6 @@ export class InstallJob implements IJob
await mkdir(path.join(downloadPath, extract_path), { recursive: true });
}
const coverResponse = await fetch(coverUrl);
const cover = Buffer.from(await coverResponse.arrayBuffer());
@ -291,7 +256,8 @@ export class InstallJob implements IJob
summary: summary,
name,
cover,
cover_type: coverResponse.headers.get('content-type')
cover_type: coverResponse.headers.get('content-type'),
metadata
};
const [{ id }] = await tx.insert(schema.games).values(game).returning({ id: schema.games.id });
@ -327,7 +293,17 @@ export class InstallJob implements IJob
}
});
} else
{
for (let i = 0; i < 10; i++)
{
cx.setProgress(i * 10, "download");
if (cx.abortSignal.aborted) return;
await sleep(1000);
}
}
events.emit('notification', { message: `${name}: Installed`, type: 'success', duration: 8000 });
}
}

View file

@ -1,13 +1,21 @@
import Elysia from "elysia";
import z, { } from "zod";
import z, { _ZodType, ZodAny, ZodObject, ZodTypeAny } from "zod";
import { taskQueue } from "../app";
import { LoginJob } from "./login-job";
import TwitchLoginJob from "./twitch-login-job";
import UpdateStoreJob from "./update-store";
import { EmulatorDownloadJob } from "./emulator-download-job";
import { getErrorMessage } from "@/bun/utils";
import { IJob } from "../task-queue";
function registerJob<const Path extends string, TS, T extends { id: Path, dataSchema?: TS; }> (_job: T, path: Path, dataSchema: TS)
function registerJob<
const Path extends string,
const Schema extends ZodTypeAny,
const States extends string,
T extends IJob<z.infer<Schema>, States>
> (_job: { id: Path; dataSchema: Schema; } & (new (...args: any[]) => T))
{
return new Elysia().ws(path, {
return new Elysia().ws(_job.id, {
body: z.discriminatedUnion('type', [
z.object({ type: z.literal('cancel') })
]),
@ -16,14 +24,14 @@ function registerJob<const Path extends string, TS, T extends { id: Path, dataSc
type: z.literal(['data', 'started', 'progress']),
status: z.string(),
progress: z.number(),
data: dataSchema
data: _job.dataSchema
}),
z.object({ type: z.literal(['completed', 'ended']) }),
z.object({ type: z.literal('error'), error: z.unknown() })
z.object({ type: z.literal(['completed', 'ended']), data: _job.dataSchema }),
z.object({ type: z.literal('error'), error: z.string() })
]),
open (ws)
{
const job = taskQueue.findJob(path);
const job = taskQueue.findJob(_job.id, _job);
if (job)
{
ws.send({ type: 'data', status: job.status, progress: job.progress, data: job.job.exposeData?.() });
@ -32,30 +40,37 @@ function registerJob<const Path extends string, TS, T extends { id: Path, dataSc
(ws.data as any).cleanup = [
taskQueue.on('started', ({ id, job }) =>
{
if (id === path)
if (id === _job.id)
{
ws.send({ type: 'started', status: job.status, progress: job.progress, data: job.job.exposeData?.() });
}
}),
taskQueue.on('progress', ({ id, job }) =>
{
if (id === path)
if (id === _job.id)
{
ws.send({ type: 'started', status: job.status, progress: job.progress, data: job.job.exposeData?.() });
}
}),
taskQueue.on('completed', ({ id }) =>
taskQueue.on('completed', ({ id, job }) =>
{
if (id === path)
if (id === _job.id)
{
ws.send({ type: 'completed' });
ws.send({ type: 'completed', data: job.job.exposeData?.() });
}
}),
taskQueue.on('ended', ({ id, job }) =>
{
if (id === _job.id)
{
ws.send({ type: 'ended', data: job.job.exposeData?.() });
}
}),
taskQueue.on('error', ({ id, error }) =>
{
if (id === path)
if (id === _job.id)
{
ws.send({ type: 'error', error: error });
ws.send({ type: 'error', error: getErrorMessage(error) });
}
})
];
@ -68,13 +83,14 @@ function registerJob<const Path extends string, TS, T extends { id: Path, dataSc
{
if (message.type === 'cancel')
{
taskQueue.findJob(path)?.abort('cancel');
taskQueue.findJob(_job.id, _job)?.abort('cancel');
}
},
});
}
export const jobs = new Elysia({ prefix: '/api/jobs' })
.use(registerJob(LoginJob, LoginJob.id, LoginJob.dataSchema))
.use(registerJob(TwitchLoginJob, TwitchLoginJob.id, TwitchLoginJob.dataSchema))
.use(registerJob(UpdateStoreJob, UpdateStoreJob.id, undefined));
.use(registerJob(LoginJob))
.use(registerJob(TwitchLoginJob))
.use(registerJob(UpdateStoreJob))
.use(registerJob(EmulatorDownloadJob));

View file

@ -1,5 +1,5 @@
import Elysia, { status } from "elysia";
import { IJob, JobContext } from "../task-queue";
import { IJob, JobBase, JobContext, JobContextFromClass } from "../task-queue";
import { LOGIN_PORT, SERVER_URL } from "@/shared/constants";
import { host, localIp } from "@/bun/utils/host";
import cors from "@elysiajs/cors";
@ -8,7 +8,7 @@ import { config } from "../app";
import z from "zod";
import { delay } from "@/shared/utils";
export class LoginJob implements IJob
export class LoginJob implements IJob<z.infer<typeof LoginJob.dataSchema>, "base">
{
endsAt: Date;
startedAt: Date;
@ -25,7 +25,7 @@ export class LoginJob implements IJob
exposeData = (): z.infer<typeof LoginJob.dataSchema> => ({ endsAt: this.endsAt, startedAt: this.startedAt, url: this.url });
async start (context: JobContext): Promise<any>
async start (context: JobContext<LoginJob, z.infer<typeof LoginJob.dataSchema>, "base">): Promise<void>
{
const loginServer = new Elysia({ serve: { hostname: localIp, port: LOGIN_PORT } })
.use(cors())

View file

@ -16,7 +16,9 @@ interface TwitchDevice
verification_uri: string;
}
export default class TwitchLoginJob implements IJob
type States = "Retrieving Device" | "Waiting For Authentication";
export default class TwitchLoginJob implements IJob<z.infer<typeof TwitchLoginJob.dataSchema>, States>
{
twitchScopes = "analytics:read:extensions analytics:read:games user:read:email";
device?: TwitchDevice;
@ -38,7 +40,7 @@ export default class TwitchLoginJob implements IJob
user_code: this.device.user_code
}) : undefined;
async start (context: JobContext): Promise<any>
async start (context: JobContext<TwitchLoginJob, z.infer<typeof TwitchLoginJob.dataSchema>, States>): Promise<any>
{
context.setProgress(0, "Retrieving Device");
let res = await fetch("https://id.twitch.tv/oauth2/device", {

View file

@ -1,12 +1,14 @@
import { ensureDir } from "fs-extra";
import { IJob, JobContext } from "../task-queue";
import { getStoreFolder } from "../store/store";
import { getStoreFolder } from "../store/services/gamesService";
import z from "zod";
export default class UpdateStoreJob implements IJob
export default class UpdateStoreJob implements IJob<never, never>
{
static id = "update-store" as const;
static origin = "https://github.com/simeonradivoev/gameflow-store.git";
static branch = "master";
static dataSchema = z.never();
async gitCommand (commands: string[], dir: string)
{
@ -40,8 +42,10 @@ export default class UpdateStoreJob implements IJob
return (await this.gitCommand(["status", "--porcelain"], dir)).length > 0;
}
async start (context: JobContext)
async start (context: JobContext<UpdateStoreJob, never, never>)
{
if (process.env.CUSTOM_STORE_PATH) return;
const storeFolder = getStoreFolder();
await ensureDir(storeFolder);
context.setProgress(10);