57 lines
1.4 KiB
JavaScript
57 lines
1.4 KiB
JavaScript
function createCSVScanner(onRow, nonJsonCols) {
|
|
let buffer = '';
|
|
|
|
return function scanChunk(chunk) {
|
|
buffer += chunk;
|
|
const lines = buffer.split('\n');
|
|
buffer = lines.pop(); // Save incomplete line
|
|
|
|
for (const line of lines) {
|
|
const fields = [];
|
|
let start = 0;
|
|
let commaCount = 0;
|
|
|
|
for (let i = 0; i < line.length; i++) {
|
|
if (line[i] === ',' && commaCount < nonJsonCols) {
|
|
fields.push(line.slice(start, i));
|
|
start = i + 1;
|
|
commaCount++;
|
|
}
|
|
}
|
|
|
|
// Push the remaining part as JSON string
|
|
const jsonStr = line.slice(start);
|
|
let jsonData = null;
|
|
try {
|
|
jsonData = JSON.parse(jsonStr);
|
|
} catch (_e) {
|
|
console.error('Invalid JSON:', jsonStr);
|
|
}
|
|
|
|
onRow(...fields, jsonData);
|
|
|
|
}
|
|
};
|
|
}
|
|
|
|
export default async function(url, nonJsonCols, onRow) {
|
|
const response = await fetch(url);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
}
|
|
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder("utf-8");
|
|
|
|
const scanner = createCSVScanner(onRow, nonJsonCols);
|
|
|
|
while (true)
|
|
{
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
const chunk = decoder.decode(value, { stream: true });
|
|
scanner(chunk);
|
|
}
|
|
}
|