Browse Source

Helper tools for snscrape and the wiki pages

master
JustAnotherArchivist 4 years ago
parent
commit
89ccd68b59
2 changed files with 83 additions and 0 deletions
  1. +49
    -0
      snscrape-upload
  2. +34
    -0
      snscrape-wiki-transfer-merge

+ 49
- 0
snscrape-upload View File

@@ -0,0 +1,49 @@
#!/bin/bash
scriptpath="$(cd "$(dirname "$0")"; pwd -P)"

insta=

if [[ -e transfer ]]
then
echo "Error: ./transfer exists" >&2
exit 1
fi

# ArchiveBot
for f in "$@"
do
upurl="$("${scriptpath}/transfer.notkiska.pw-upload" "${f}")"
echo "${upurl}" >&3
if [[ "${f}" == facebook-* ]]
then
echo "!ao < ${upurl}"
elif [[ "${f}" == twitter-* ]]
then
echo "!ao < ${upurl} --concurrency 6 --delay 0"
elif [[ "${f}" == instagram-* ]]
then
echo "!a < ${upurl}"
insta=1
else
echo "!ao < ${upurl}"
fi
done 3>transfer

# chromebot
for f in "$@"
do
if [[ "${f}" == facebook-@* || "${f}" == twitter-@* ]]
then
head -1 "${f}"
elif [[ "${f}" == twitter-#* ]]
then
head -4 "${f}"
fi
done | sed 's,^,chromebot: a ,'

# Instagram ignore warning
if [[ "${insta}" ]]
then
echo "Don't forget to add the Instagram ignore! ^https?://www.instagram.com/.*[?&]hl=" >&2
fi
echo "Wrote ./transfer, you can run snscrape-wiki-transfer-merge now if ./wiki exists." >&2

+ 34
- 0
snscrape-wiki-transfer-merge View File

@@ -0,0 +1,34 @@
#!/bin/bash
if [[ ! -e wiki || ! -e transfer ]]
then
echo "Error: ./wiki or ./transfer missing"
exit 1
fi

while read -r line
do
if [[ "${line}" == *facebook* ]]
then
l="${line%/}"
search="facebook.*${l##*/}"
elif [[ "${line}" == *twitter* ]]
then
search="twitter.*${line##*/}"
elif [[ "${line}" == *instagram* ]]
then
l="${line%/}"
search="instagram.*${l##*/}"
else
search=
fi

if [[ "${search}" ]]
then
if grep -q "${search}" transfer
then
echo "$(grep "${search}" transfer) | note = ${line}"
continue
fi
fi
echo "${line}"
done <wiki

Loading…
Cancel
Save