Upload files to "/"
This commit is contained in:
@@ -0,0 +1,311 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||||
<title>CSV File Search</title>
|
||||
<style>
|
||||
body { font-family: system-ui, Arial, sans-serif; max-width: 900px; margin: 24px auto; padding: 0 12px; }
|
||||
.row { display: flex; gap: 10px; align-items: center; flex-wrap: wrap; }
|
||||
input[type="text"] { flex: 1; min-width: 260px; padding: 10px; }
|
||||
button { padding: 10px 14px; cursor: pointer; }
|
||||
pre { background: #f6f7f9; padding: 12px; overflow: auto; border-radius: 8px; }
|
||||
.meta { color: #555; margin-top: 6px; font-size: 0.95em; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>CSV File Search</h1>
|
||||
|
||||
<div class="row">
|
||||
<label>
|
||||
<div style="margin-bottom:6px;">Upload CSV files</div>
|
||||
<input id="csvFiles" type="file" accept=".csv,text/csv" multiple />
|
||||
</label>
|
||||
|
||||
<button id="loadBtn" disabled>Load CSVs</button>
|
||||
</div>
|
||||
|
||||
<div class="meta" id="status">Status: waiting for files.</div>
|
||||
|
||||
<hr style="margin: 18px 0;" />
|
||||
|
||||
<div class="row">
|
||||
<input id="queryBox" type="text" placeholder="Type a search term (e.g., john)..." disabled />
|
||||
<button id="searchBtn" disabled>Search</button>
|
||||
<button id="listColumnsBtn" disabled>List columns</button>
|
||||
</div>
|
||||
|
||||
<p class="meta" id="hint">
|
||||
Searches all “text-like” columns in all loaded CSVs using a substring match.
|
||||
</p>
|
||||
|
||||
<h2 style="margin-top: 18px;">Results</h2>
|
||||
<pre id="out">No results yet.</pre>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/papaparse@5.4.1/papaparse.min.js"></script>
|
||||
<script>
|
||||
// datasets: array of { file, fileName, headers, textTargets:Set(headerName) }
|
||||
let datasets = [];
|
||||
|
||||
const statusEl = document.getElementById('status');
|
||||
const outEl = document.getElementById('out');
|
||||
const csvFilesEl = document.getElementById('csvFiles');
|
||||
const loadBtn = document.getElementById('loadBtn');
|
||||
const queryBox = document.getElementById('queryBox');
|
||||
const searchBtn = document.getElementById('searchBtn');
|
||||
const listColumnsBtn = document.getElementById('listColumnsBtn');
|
||||
|
||||
function setStatus(msg) { statusEl.textContent = "Status: " + msg; }
|
||||
|
||||
function detectTextLikeColumnsFromSample(sampleRows, headers, sampleSize = 200) {
|
||||
const limit = Math.min(sampleRows.length, sampleSize);
|
||||
|
||||
// Stats indexed by column index (so duplicates/empties don't break alignment)
|
||||
const colStats = new Array(headers.length).fill(null).map(() => ({ nonEmpty: 0, numericLike: 0 }));
|
||||
|
||||
const numericRegex = /^-?\d+(\.\d+)?$/;
|
||||
|
||||
for (let r = 0; r < limit; r++) {
|
||||
const row = sampleRows[r];
|
||||
|
||||
for (let c = 0; c < headers.length; c++) {
|
||||
const h = headers[c];
|
||||
if (!h) continue; // ignore empty header names for "textTargets" selection
|
||||
|
||||
const v = (row[c] ?? '').toString().trim();
|
||||
if (!v) continue;
|
||||
|
||||
colStats[c].nonEmpty++;
|
||||
if (numericRegex.test(v)) colStats[c].numericLike++;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep a Set of header names for UI and matching selection (matches may include duplicate names)
|
||||
const targets = new Set();
|
||||
|
||||
for (let c = 0; c < headers.length; c++) {
|
||||
const h = headers[c];
|
||||
if (!h) continue;
|
||||
|
||||
const s = colStats[c];
|
||||
if (s.nonEmpty === 0) continue;
|
||||
|
||||
const numericRatio = s.numericLike / Math.max(1, s.nonEmpty);
|
||||
// numericRatio < 0.8 => not mostly-numeric => treat as text-like
|
||||
if (numericRatio < 0.8) targets.add(h);
|
||||
}
|
||||
|
||||
return targets;
|
||||
}
|
||||
|
||||
csvFilesEl.addEventListener('change', () => {
|
||||
loadBtn.disabled = !csvFilesEl.files?.length;
|
||||
});
|
||||
|
||||
loadBtn.addEventListener('click', async () => {
|
||||
const files = [...(csvFilesEl.files || [])];
|
||||
if (!files.length) return;
|
||||
|
||||
try {
|
||||
outEl.textContent = "Loading CSVs...";
|
||||
setStatus(`loading ${files.length} file(s)...`);
|
||||
datasets = [];
|
||||
|
||||
// Load sequentially so you don't spawn multiple heavy parses at once
|
||||
const SAMPLE_SIZE = 200;
|
||||
|
||||
for (let fi = 0; fi < files.length; fi++) {
|
||||
const file = files[fi];
|
||||
setStatus(`sampling ${fi + 1}/${files.length}: ${file.name}`);
|
||||
outEl.textContent = `Sampling: ${file.name}`;
|
||||
|
||||
// Pass 1: stream to get headers + first N data rows
|
||||
const sampleRows = [];
|
||||
let headers = null;
|
||||
let headerSet = false;
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
Papa.parse(file, {
|
||||
header: false,
|
||||
skipEmptyLines: true,
|
||||
worker: true,
|
||||
chunkSize: 1024 * 1024,
|
||||
step: function(results, parser) {
|
||||
if (!headerSet) {
|
||||
// First row is header row
|
||||
headers = results.data.map(h => (h ?? '').toString().trim());
|
||||
// IMPORTANT: do NOT filter empty header names; keep positions stable
|
||||
headerSet = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (headers && sampleRows.length < SAMPLE_SIZE) {
|
||||
sampleRows.push(results.data);
|
||||
}
|
||||
|
||||
if (sampleRows.length >= SAMPLE_SIZE) {
|
||||
parser.abort(); // only need a sample for column detection
|
||||
}
|
||||
},
|
||||
complete: function() {
|
||||
if (!headers || !headers.length) return resolve();
|
||||
const textTargets = detectTextLikeColumnsFromSample(sampleRows, headers, SAMPLE_SIZE);
|
||||
datasets.push({ file, fileName: file.name, headers, textTargets });
|
||||
resolve();
|
||||
},
|
||||
error: function(err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
queryBox.disabled = false;
|
||||
searchBtn.disabled = false;
|
||||
listColumnsBtn.disabled = false;
|
||||
|
||||
const totalCols = datasets.reduce((acc, d) => acc + d.textTargets.size, 0);
|
||||
setStatus("loaded.");
|
||||
outEl.textContent =
|
||||
`Loaded ${datasets.length} CSV(s). Total text-like columns (sum of unique names): ${totalCols}.`;
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
setStatus("failed.");
|
||||
outEl.textContent = "Error:\n" + e.message;
|
||||
}
|
||||
});
|
||||
|
||||
listColumnsBtn.addEventListener('click', () => {
|
||||
if (!datasets.length) return;
|
||||
|
||||
const lines = [];
|
||||
for (const d of datasets) {
|
||||
const cols = [...d.textTargets];
|
||||
if (!cols.length) continue;
|
||||
lines.push(`${d.fileName} (${cols.length}):\n- ${cols.join("\n- ")}`);
|
||||
}
|
||||
|
||||
if (!lines.length) {
|
||||
outEl.textContent = "No text-like columns detected in any loaded CSV.";
|
||||
return;
|
||||
}
|
||||
|
||||
outEl.textContent = "Text-like columns:\n\n" + lines.join("\n\n");
|
||||
});
|
||||
|
||||
searchBtn.addEventListener('click', async () => {
|
||||
const term = (queryBox.value || '').trim();
|
||||
if (!term) {
|
||||
outEl.textContent = "Type a search term first.";
|
||||
return;
|
||||
}
|
||||
|
||||
if (!datasets.length) return;
|
||||
|
||||
const termLower = term.toLowerCase();
|
||||
const LIMIT = 1000;
|
||||
const results = [];
|
||||
|
||||
// Build a quick lookup for headers by file name
|
||||
const headersByFile = new Map(datasets.map(d => [d.fileName, d.headers]));
|
||||
|
||||
try {
|
||||
setStatus(`searching for "${term}"...`);
|
||||
outEl.textContent = `Searching: "${term}"`;
|
||||
|
||||
// Search each dataset by streaming the entire file (second pass per file)
|
||||
for (let di = 0; di < datasets.length; di++) {
|
||||
const d = datasets[di];
|
||||
if (!d.textTargets.size) continue;
|
||||
|
||||
setStatus(`searching ${di + 1}/${datasets.length}: ${d.fileName}`);
|
||||
|
||||
// Build candidate indices by header position (supports duplicate header names)
|
||||
const targetIndices = [];
|
||||
for (let i = 0; i < d.headers.length; i++) {
|
||||
const h = d.headers[i];
|
||||
if (h && d.textTargets.has(h)) targetIndices.push(i);
|
||||
}
|
||||
|
||||
let dataRowIndex = 0;
|
||||
let sawHeader = false;
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
Papa.parse(d.file, {
|
||||
header: false,
|
||||
skipEmptyLines: true,
|
||||
worker: true,
|
||||
chunkSize: 1024 * 1024,
|
||||
step: function(results2, parser) {
|
||||
if (!sawHeader) {
|
||||
sawHeader = true;
|
||||
return; // skip header row
|
||||
}
|
||||
|
||||
// This is a data row (since we skipped the header row above)
|
||||
dataRowIndex++;
|
||||
|
||||
// If any target cell in this row matches, store the whole row once
|
||||
let rowMatched = false;
|
||||
|
||||
for (const ci of targetIndices) {
|
||||
const v = (results2.data?.[ci] ?? '').toString().trim();
|
||||
if (v && v.toLowerCase().includes(termLower)) {
|
||||
rowMatched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (rowMatched) {
|
||||
results.push({
|
||||
file_name: d.fileName,
|
||||
row_index: dataRowIndex + 1, // header row is row 1; data starts at 2
|
||||
row: results2.data
|
||||
});
|
||||
|
||||
if (results.length >= LIMIT) {
|
||||
parser.abort();
|
||||
}
|
||||
}
|
||||
},
|
||||
complete: function() { resolve(); },
|
||||
error: function(err) { reject(err); }
|
||||
});
|
||||
});
|
||||
|
||||
if (results.length >= LIMIT) break;
|
||||
|
||||
// Update UI occasionally between files
|
||||
outEl.textContent =
|
||||
`Searching: "${term}"\n\nCollected ${results.length} matched row(s) so far (up to ${LIMIT}).`;
|
||||
}
|
||||
|
||||
if (!results.length) {
|
||||
outEl.textContent = `No matches found for "${term}".`;
|
||||
} else {
|
||||
const lines = results.map(x => {
|
||||
const headers = headersByFile.get(x.file_name) || [];
|
||||
const pairs = x.row.map((val, i) => {
|
||||
const key = headers[i] ?? `col_${i + 1}`;
|
||||
const safeVal = (val ?? '').toString();
|
||||
return `${key}: ${safeVal}`;
|
||||
});
|
||||
return `${x.file_name} (row ${x.row_index}):\n ${pairs.join("\n ")}`;
|
||||
});
|
||||
|
||||
outEl.textContent =
|
||||
`Found ${results.length} matched row(s) for "${term}" (showing up to ${LIMIT}):\n\n` +
|
||||
lines.join("\n\n");
|
||||
}
|
||||
|
||||
setStatus("done.");
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
setStatus("failed.");
|
||||
outEl.textContent = "Error:\n" + e.message;
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
Reference in New Issue
Block a user