|
// Dat modules |
|
var storage = require('dat-storage'); |
|
var hyperdrive = require('hyperdrive'); |
|
var hypercoreProtocol = require('hypercore-protocol'); |
|
var discoverySwarm = require('discovery-swarm'); |
|
var swarmDefaults = require('dat-swarm-defaults'); |
|
var Dat = require('dat-node'); |
|
|
|
|
|
// =========================================================================== |
|
function initSwarm(port) { |
|
let swarm = discoverySwarm(swarmDefaults({ |
|
hash: false, |
|
stream: function(info) { return replicate(info); }, |
|
})); |
|
|
|
swarm.listen(port); |
|
|
|
swarm.on("error", function(err) { console.log(err); }); |
|
swarm.on("connection", function(conn, info) { |
|
// not used, stream callback called instead? |
|
//console.log(`Conn Channel: ${info.channel.toString('hex')}`); |
|
//console.log(`Conn Id: ${info.id.toString('hex')}`); |
|
//console.log(conn); |
|
//console.log(info); |
|
}); |
|
|
|
swarm.on("peer", function(peer) { console.debug(`Peer: ${peer.id}`); }); |
|
return swarm; |
|
} |
|
|
|
|
|
// =========================================================================== |
|
function shareDir(fullDir, swarm, allDats, dontUpdate) { |
|
console.log(`Share Dir: ${fullDir}`); |
|
|
|
// if not dontUpdate, auto init Dat if needed and importFiles, then add to swarm |
|
if (!dontUpdate) { |
|
Dat(fullDir, function(err, dat) { |
|
if (err) { |
|
throw err; |
|
} |
|
|
|
let importer = dat.importFiles(); |
|
importer.on('end', function() { |
|
addToSwarm(dat.archive); |
|
}); |
|
}); |
|
|
|
// else, only add existing add existing dat/hyperdrive |
|
} else { |
|
let opts = {"latest": true, |
|
"indexing": true, |
|
"dir": fullDir |
|
} |
|
|
|
let datDrive = hyperdrive(storage(fullDir), null, opts); |
|
datDrive.on("ready", function() { addToSwarm(datDrive); }); |
|
} |
|
|
|
|
|
function addToSwarm(datDrive) { |
|
let dk = datDrive.discoveryKey.toString('hex'); |
|
let key = datDrive.key.toString('hex'); |
|
|
|
console.log(`Sharing DAT: ${key}`); |
|
//console.log(`Sharing Discovery Key: ${dk}`); |
|
|
|
// map discoveryKey hex -> hyperdrive |
|
allDats[dk] = datDrive; |
|
|
|
swarm.join(datDrive.discoveryKey, { announce: true }, function() { |
|
console.log(`Added discoveryKey to swarm: ${dk}`); |
|
}); |
|
} |
|
} |
|
|
|
// =========================================================================== |
|
function replicate(info) { |
|
let stream = hypercoreProtocol({ |
|
live: true, |
|
encrypt: true |
|
}); |
|
|
|
stream.on('error', function(err) { |
|
console.log(`Stream Error: ${err}`); |
|
}); |
|
|
|
stream.on('close', function() { |
|
console.log('Closed Stream'); |
|
}); |
|
|
|
stream.on('end', function() { |
|
console.log('Done Uploading'); |
|
}); |
|
|
|
stream.on('feed', function(discoveryKey) { |
|
var dk = discoveryKey.toString('hex') |
|
|
|
console.log(`Feed ${dk}`); |
|
|
|
var datDrive = allDats[dk]; |
|
|
|
if (datDrive) { |
|
console.log("DAT found, uploading..."); |
|
datDrive.replicate({stream: stream, |
|
live: false, |
|
upload: true, |
|
download: false}); |
|
} else { |
|
console.log(`Dat Not Found (discoveryKey: ${dk})`); |
|
} |
|
}); |
|
|
|
return stream; |
|
} |
|
|
|
|
|
// =========================================================================== |
|
// Sample Usage |
|
var DEFAULT_PORT = 3282; |
|
|
|
var ROOT_DIR = "/path/to/webrecorder/data/storage/"; |
|
var path = require('path'); |
|
|
|
var allDats = {}; |
|
|
|
var swarm = initSwarm(DEFAULT_PORT); |
|
|
|
swarm.on("listening", function() { |
|
console.log("Swarm Listening..."); |
|
|
|
// Add DAT Collections Here... |
|
shareDir(path.join(ROOT_DIR, '2018-06-01/qqpsxtybtwaxlwsg'), swarm, allDats); |
|
shareDir(path.join(ROOT_DIR, '2018-05-16/abccq36y2aowhw7q'), swarm, allDats); |
|
shareDir(path.join(ROOT_DIR, '2018-06-11/sdkauabe3hmta4zl'), swarm, allDats); |
|
}); |
|
|
|
|
|
|
Basic Usage:
If setting ROOT_DIR to path to Webrecorder
/data/storageCan then add each collection that should be shared via dat with shareDir()
By default, each specified collection is auto-inited with new dat if one doesn't already exist.
Will output
Sharing DAT: $keyfor each key added.While running, should be able to clone $key via
dat clone $keyon a different machine with dat installed. (On same machine, DAT is smart enough to just copy the local feed directly).