mteb-turkish / mteb-visualizer-react /scripts /generate_manifest.js
alibayram's picture
27 models and 26 tasks done
60b40e0
Raw History Blame Contribute Delete
3.58 kB
import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const PUBLIC_DIR = path.join(__dirname, "../public");
const RESULTS_DIR = path.join(PUBLIC_DIR, "results");
const MANIFEST_FILE = path.join(PUBLIC_DIR, "models_manifest.json");
function generateManifest() {
console.log("Generating models_manifest.json...");
if (!fs.existsSync(RESULTS_DIR)) {
console.error(`Results directory not found: ${RESULTS_DIR}`);
process.exit(1);
}
const manifest = {};
const modelDirs = fs.readdirSync(RESULTS_DIR).filter((entry) => {
return fs.statSync(path.join(RESULTS_DIR, entry)).isDirectory();
});
for (const modelName of modelDirs) {
const modelPath = path.join(RESULTS_DIR, modelName);
const revisionDirs = fs.readdirSync(modelPath).filter((entry) => {
// Ignore .DS_Store and other hidden files/dirs just in case
return (
!entry.startsWith(".") &&
fs.statSync(path.join(modelPath, entry)).isDirectory()
);
});
if (revisionDirs.length === 0) {
console.warn(`No revision directory found for model: ${modelName}`);
continue;
}
// Identify the valid revision.
// Logic: If there's one, take it. If multiple, take the first one (or we could sort by date/name).
// The current manual manifest seems to rely on specific revisions, assuming mostly 1 exists or the user picks one.
// For auto-generation, taking the most recent or simply the first found is a reasonable default.
// Let's take the first one for now, as usually there is only one "active" processed result set in this structure.
const revision = revisionDirs[0];
let revisionPath = path.join(modelPath, revision);
// Some models (e.g. fasttext, glove, text-embedding-3-small) have a double-nested
// structure: model/no_model_name_available/no_revision_available/tasks.json
// If the revision dir contains no JSON files but has a single subdirectory, descend into it.
let actualRelativePrefix = `results/${modelName}/${revision}`;
const revisionContents = fs
.readdirSync(revisionPath)
.filter((e) => !e.startsWith("."));
const hasJsonFiles = revisionContents.some((f) => f.endsWith(".json"));
if (!hasJsonFiles) {
const subDirs = revisionContents.filter((e) =>
fs.statSync(path.join(revisionPath, e)).isDirectory(),
);
if (subDirs.length === 1) {
const subDir = subDirs[0];
revisionPath = path.join(revisionPath, subDir);
actualRelativePrefix = `results/${modelName}/${revision}/${subDir}`;
console.log(
` Model "${modelName}" has nested structure, descending into "${subDir}"`,
);
}
}
const tasks = {};
const taskFiles = fs
.readdirSync(revisionPath)
.filter((file) => file.endsWith(".json") && file !== "model_meta.json");
for (const taskFile of taskFiles) {
const taskName = path.basename(taskFile, ".json");
// Path must be relative to public/ - e.g. results/model/revision/task.json
const relativePath = `${actualRelativePrefix}/${taskFile}`;
tasks[taskName] = relativePath;
}
manifest[modelName] = {
revision: revision,
tasks: tasks,
};
}
const jsonContent = JSON.stringify(manifest, null, 2);
fs.writeFileSync(MANIFEST_FILE, jsonContent);
console.log(
`Successfully generated manifest with ${Object.keys(manifest).length} models.`,
);
}
generateManifest();