Last active
September 12, 2025 04:09
-
-
Save un4ckn0wl3z/cdac58f45fc7cb5698225031dd183687 to your computer and use it in GitHub Desktop.
stream_readable.js
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| const http = require('http'); | |
| // Sample predefined JSON array (100 records for example) | |
| const data = Array.from({ length: 100000 }, (_, i) => ({ | |
| id: i + 1, | |
| name: `Item${i + 1}`, | |
| value: Math.random().toFixed(4) | |
| })); | |
| // Async generator to yield JSON array chunks as NDJSON | |
| async function* generateJsonChunks(jsonArray, chunkSize = 10) { | |
| for (let i = 0; i < jsonArray.length; i += chunkSize) { | |
| // Take a chunk of the array | |
| const chunk = jsonArray.slice(i, i + chunkSize); | |
| // Convert to NDJSON: each object on a new line | |
| const chunkString = chunk.map(item => JSON.stringify(item) + '\n').join(''); | |
| yield Buffer.from(chunkString); // Yield as Buffer for efficiency | |
| // Optional: Simulate real-time processing (remove if not needed) | |
| await new Promise(resolve => setTimeout(resolve, 10)); | |
| } | |
| } | |
| // Alternative: Full JSON array (not NDJSON, yields entire array in chunks) | |
| async function* generateFullJson(jsonArray, chunkSize = 1024) { | |
| const jsonString = JSON.stringify(jsonArray); // Full array as string | |
| for (let i = 0; i < jsonString.length; i += chunkSize) { | |
| const chunk = jsonString.slice(i, i + chunkSize); | |
| yield Buffer.from(chunk); | |
| } | |
| } | |
| const server = http.createServer((req, res) => { | |
| if (req.url === '/stream-json' && req.method === 'GET') { | |
| // Set headers for NDJSON streaming | |
| res.writeHead(200, { | |
| 'Content-Type': 'application/x-ndjson', // Use application/json for full JSON | |
| 'Transfer-Encoding': 'chunked', | |
| 'Content-Disposition': 'attachment; filename="data.jsonl"' | |
| }); | |
| // Use NDJSON generator (swap to generateFullJson for full JSON) | |
| const { Readable } = require('stream'); | |
| const readable = Readable.from(generateJsonChunks(data, 10)); | |
| readable.pipe(res); | |
| // Handle errors | |
| readable.on('error', (err) => { | |
| console.error('Stream error:', err); | |
| if (!res.headersSent) { | |
| res.statusCode = 500; | |
| res.end('Internal Server Error'); | |
| } | |
| }); | |
| } else { | |
| res.writeHead(404); | |
| res.end('Not Found'); | |
| } | |
| }); | |
| // Start server | |
| server.listen(3000, () => { | |
| console.log('Server running at http://localhost:3000/stream-json'); | |
| }); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment