From 72818019bca249d22ab55461507a9c279a369566 Mon Sep 17 00:00:00 2001
From: JustAnotherArchivist <JustAnotherArchivist@users.noreply.github.com>
Date: Thu, 16 May 2019 22:54:45 +0000
Subject: [PATCH] Extract external links from Twitter

---
 snscrape-twitter-hashtag | 2 +-
 snscrape-twitter-user    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/snscrape-twitter-hashtag b/snscrape-twitter-hashtag
index 0abb7fa..2869dff 100755
--- a/snscrape-twitter-hashtag
+++ b/snscrape-twitter-hashtag
@@ -6,5 +6,5 @@ shift
 	echo "https://twitter.com/hashtag/${hashtag}?src=hash"
 	echo "https://twitter.com/hashtag/${hashtag}?f=tweets&vertical=default"
 	echo "https://twitter.com/hashtag/${hashtag}?f=tweets&vertical=default&src=hash"
-	snscrape "$@" twitter-hashtag "${hashtag}"
+	snscrape --format '{url} {tcooutlinksss} {outlinksss}' "$@" twitter-hashtag "${hashtag}" | tr ' ' '\n' | grep -v '^$' | awk '!seen[$0]++'
 } > "twitter-#${hashtag}"
diff --git a/snscrape-twitter-user b/snscrape-twitter-user
index de86688..9134076 100755
--- a/snscrape-twitter-user
+++ b/snscrape-twitter-user
@@ -7,6 +7,6 @@ then
 	if [[ "${user}" != "${origUser}" ]]; then echo "Username fix: ${origUser} -> ${user}" >&2; fi
 	{
 		echo "https://twitter.com/${user}"
-		snscrape "$@" twitter-user "${user}"
-	} > "twitter-@${user}"
+		snscrape --format '{url} {tcooutlinksss} {outlinksss}' "$@" twitter-user "${user}"
+	} | tr ' ' '\n' | grep -v '^$' | awk '!seen[$0]++' > "twitter-@${user}"
 fi