Verified Commit b375d3b9 authored by Michal 'vorner' Vaner's avatar Michal 'vorner' Vaner
Browse files

Adjust the scripts to use compiled version

Testing needed.
parent a1ecc31b
......@@ -6,8 +6,11 @@ set -e
DIR=$(dirname "$0")
if [ '!' -x "$DIR/process/target/release/process" ] ; then
echo "Please compile and provide the process executable"
exit 1
fi
FILES=$(ls *.csv)
cat $FILES | LC_ALL=C sort -S 2G -T . >sorted
"$DIR/jsonize.pl" <sorted >jsonized
"$DIR/to_db.pl" -d "$HOME/archivist.ini" <jsonized
rm $FILES sorted jsonized
"$DIR/process/target/release/process" $FILES | "$DIR/to_db.pl" -d "$HOME/db.ini"
rm $FILES
#!/bin/sh
# Import the data into database. It does it all compressed during the way
# (because there's a lot of data) and uses many cores during the processing.
# Import the data into database.
set -e
DIR=$(dirname "$0")
(
pbzip2 -d < telnet.csv.bz2
pbzip2 -d < ssh.csv.bz2
pbzip2 -d < firewall.csv.bz2
pbzip2 -d < firewall_all.csv.bz2
) | LC_ALL=C sort --compress-prog="$DIR/bzchoose" -T . -S 4G | "$DIR/split.pl"
cat >Makefile <<ENDMAKE
INPUTS:=\$(wildcard split/*.csv.gz)
OUTPUTS:=\$(patsubst %.csv.gz,%.json.gz,\$(INPUTS))
if [ '!' -x "$DIR/process/target/release/process" ] ; then
echo "Please compile and provide the process executable"
exit 1
fi
all: \$(OUTPUTS)
%.json.gz: %.csv.gz
gunzip -c < \$< | "$DIR/jsonize.pl" | gzip -1 >\$@
ENDMAKE
make -j12
for i in split/*.json.gz ; do
gunzip <"$i"
done | "$DIR/to_db.pl" -i -d "$HOME/db.ini"
"$DIR/process/target/release/process" telnet.csv.bz2 ssh.csv.bz2 firewall.csv.bz2 | "$DIR/to_db.pl" -i -d "$HOME/db.ini"
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment