Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Remove Twitter stuff
Uku Taht4d0bc61ffdd1
diff --git a/assets/css/app.css b/assets/css/app.cssindex 708b842eb9..3fc1397683 100644--- a/assets/css/app.css+++ b/assets/css/app.css@@ -131,6 +131,7 @@ blockquote {animation: pulse-ring 3s cubic-bezier(0.215, 0.61, 0.355, 1) infinite;@apply bg-green-500;}+.pulsating-circle::after {content: '';position: absolute;@@ -150,9 +151,11 @@ blockquote {0% {transform: scale(.33);}+50% {transform: scale(1);}+40%, 100% {opacity: 0;}@@ -162,9 +165,11 @@ blockquote {0% {transform: scale(.8);}+25% {transform: scale(1);}+50%, 100% {transform: scale(.8);}@@ -187,11 +192,13 @@ blockquote {border: 8px solid transparent;border-bottom-color: rgba(27,31,35,0.15);}+.dropdown-content::before, .dropdown-content::after {position: absolute;display: inline-block;content: "";}+.dropdown-content::after {top: -14px;right: 9px;@@ -226,24 +233,6 @@ blockquote {background-color: rgb(26, 32, 44);}-.twitter-icon {- width: 1.25em;- height: 1.25em;- display: inline-block;- background-repeat: no-repeat;- background-size: contain;- vertical-align: text-bottom;- background-image: url(data:image/svg+xml;charset=utf-8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%2072%2072%22%3E%3Cpath%20fill%3D%22none%22%20d%3D%22M0%200h72v72H0z%22%2F%3E%3Cpath%20class%3D%22icon%22%20fill%3D%22%231da1f2%22%20d%3D%22M68.812%2015.14c-2.348%201.04-4.87%201.744-7.52%202.06%202.704-1.62%204.78-4.186%205.757-7.243-2.53%201.5-5.33%202.592-8.314%203.176C56.35%2010.59%2052.948%209%2049.182%209c-7.23%200-13.092%205.86-13.092%2013.093%200%201.026.118%202.02.338%202.98C25.543%2024.527%2015.9%2019.318%209.44%2011.396c-1.125%201.936-1.77%204.184-1.77%206.58%200%204.543%202.312%208.552%205.824%2010.9-2.146-.07-4.165-.658-5.93-1.64-.002.056-.002.11-.002.163%200%206.345%204.513%2011.638%2010.504%2012.84-1.1.298-2.256.457-3.45.457-.845%200-1.666-.078-2.464-.23%201.667%205.2%206.5%208.985%2012.23%209.09-4.482%203.51-10.13%205.605-16.26%205.605-1.055%200-2.096-.06-3.122-.184%205.794%203.717%2012.676%205.882%2020.067%205.882%2024.083%200%2037.25-19.95%2037.25-37.25%200-.565-.013-1.133-.038-1.693%202.558-1.847%204.778-4.15%206.532-6.774z%22%2F%3E%3C%2Fsvg%3E);-}--.tweet-text a {- @apply text-blue-500;-}--.tweet-text a:hover {- text-decoration: underline;-}-.stats-item {min-height: 436px;}@@ -258,7 +247,7 @@ blockquote {height: 27.25rem;}- .stats-item__header {+ .stats-item-header {height: inherit;}}@@ -274,6 +263,7 @@ blockquote {.fade-enter {opacity: 0;}+.fade-enter-active {opacity: 1;transition: opacity 100ms ease-in;@@ -335,4 +325,4 @@ iframe[hidden] {text-decoration-color: #6366F1; /* tailwind's indigo-500 */}-}\ No newline at end of file+}diff --git a/assets/js/dashboard/date.js b/assets/js/dashboard/date.jsindex 477307686b..dd8caa1e68 100644--- a/assets/js/dashboard/date.js+++ b/assets/js/dashboard/date.js@@ -44,11 +44,6 @@ export function formatDayShort(date) {return `${date.getDate()} ${formatMonth(date).substring(0, 3)}`;}-export function formatFullDate(date) {- const shortDate = formatMonth(date).substring(0, 3)- return `${shortDate} ${date.getDate()}, ${date.getFullYear()}`;-}-export function parseUTCDate(dateString) {var date = new Date(dateString);return new Date(date.getTime() + date.getTimezoneOffset() * 60000);diff --git a/assets/js/dashboard/stats/devices/index.js b/assets/js/dashboard/stats/devices/index.jsindex 1e5ce88f2e..e993ede308 100644--- a/assets/js/dashboard/stats/devices/index.js+++ b/assets/js/dashboard/stats/devices/index.js@@ -184,7 +184,7 @@ export default class Devices extends React.Component {className="stats-item flex flex-col mt-6 stats-item--has-header w-full"><div- className="stats-item__header flex flex-col flex-grow relative p-4 bg-white rounded shadow-xl dark:bg-gray-825"+ className="stats-item-header flex flex-col flex-grow relative p-4 bg-white rounded shadow-xl dark:bg-gray-825"><div className="flex justify-between w-full"><h3 className="font-bold dark:text-gray-100">Devices</h3>diff --git a/assets/js/dashboard/stats/locations/index.js b/assets/js/dashboard/stats/locations/index.jsindex 6a2c1a6897..bb69f476fe 100644--- a/assets/js/dashboard/stats/locations/index.js+++ b/assets/js/dashboard/stats/locations/index.js@@ -137,7 +137,7 @@ export default class Locations extends React.Component {className="stats-item flex flex-col w-full mt-6 stats-item--has-header"><div- className="stats-item__header flex flex-col flex-grow bg-white dark:bg-gray-825 shadow-xl rounded p-4 relative"+ className="stats-item-header flex flex-col flex-grow bg-white dark:bg-gray-825 shadow-xl rounded p-4 relative"><div className="w-full flex justify-between"><h3 className="font-bold dark:text-gray-100">diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.jsindex a38a55aa9c..459d97c1e4 100644--- a/assets/js/dashboard/stats/modals/referrer-drilldown.js+++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js@@ -5,7 +5,6 @@ import Modal from './modal'import * as api from '../../api'import numberFormatter, {durationFormatter} from '../../number-formatter'import {parseQuery, toHuman} from '../../query'-import {formatFullDate} from '../../date'class ReferrerDrilldownModal extends React.Component {constructor(props) {@@ -84,68 +83,21 @@ class ReferrerDrilldownModal extends React.Component {)}- renderTweet(tweet, index) {- const authorUrl = `https://twitter.com/${tweet.author_handle}`- const tweetUrl = `${authorUrl}/status/${tweet.tweet_id}`- const border = index === 0 ? '' : ' pt-4 border-t border-gray-300 dark:border-gray-500'-+ renderReferrer(referrer) {return (- <div key={tweet.tweet_id}>- <div className={"flex items-center my-4" + border} >- <a className="flex items-center group" href={authorUrl} target="_blank" rel="noreferrer">- <img className="rounded-full w-8" src={tweet.author_image} />- <div className="ml-2 leading-tight">- <div className="font-bold group-hover:text-blue-500">{tweet.author_name}</div>- <div className="text-xs text-gray-500 dark:text-gray-400">@{tweet.author_handle}</div>- </div>- </a>- <a className="ml-auto twitter-icon" href={tweetUrl} target="_blank" rel="noreferrer"></a>- </div>- <div className="my-2 cursor-text tweet-text whitespace-pre-wrap" dangerouslySetInnerHTML={{__html: tweet.text}}>- </div>- <div className="text-xs text-gray-700 dark:text-gray-300 font-medium">- {formatFullDate(new Date(tweet.created))}- </div>- </div>+ <tr className="text-sm dark:text-gray-200" key={referrer.name}>+ <td className="p-2">+ { this.renderReferrerName(referrer) }+ </td>+ {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.total_visitors)}</td> }+ <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.visitors)}</td>+ {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(referrer)}</td> }+ {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatDuration(referrer)}</td> }+ {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{referrer.conversion_rate}%</td> }+ </tr>)}- renderReferrer(referrer) {- if (referrer.tweets) {- return (- <tr className="text-sm dark:text-gray-200" key={referrer.name}>- <td className="p-2">- { this.renderReferrerName(referrer) }- <span className="text-gray-500 ml-2 text-xs">- appears in {referrer.tweets.length} tweets- </span>- <div className="my-4 pl-4 border-l-2 border-gray-300 dark:border-gray-500">- { referrer.tweets.map(this.renderTweet) }- </div>- </td>- {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.total_visitors)}</td> }- <td className="p-2 w-32 font-medium" align="right" valign="top">{numberFormatter(referrer.visitors)}</td>- {this.showExtra() && <td className="p-2 w-32 font-medium" align="right" valign="top">{this.formatBounceRate(referrer)}</td> }- {this.showExtra() && <td className="p-2 w-32 font-medium" align="right" valign="top">{this.formatDuration(referrer)}</td> }- {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{referrer.conversion_rate}%</td> }- </tr>- )- } else {- return (- <tr className="text-sm dark:text-gray-200" key={referrer.name}>- <td className="p-2">- { this.renderReferrerName(referrer) }- </td>- {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.total_visitors)}</td> }- <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.visitors)}</td>- {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(referrer)}</td> }- {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatDuration(referrer)}</td> }- {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{referrer.conversion_rate}%</td> }- </tr>- )- }- }-renderGoalText() {if (this.state.query.filters.goal) {return (diff --git a/assets/js/dashboard/stats/pages/index.js b/assets/js/dashboard/stats/pages/index.jsindex 990a4767c3..ac1ee2cff3 100644--- a/assets/js/dashboard/stats/pages/index.js+++ b/assets/js/dashboard/stats/pages/index.js@@ -70,7 +70,7 @@ export default class Pages extends React.Component {className="stats-item flex flex-col w-full mt-6 stats-item--has-header"><div- className="stats-item__header flex flex-col flex-grow bg-white dark:bg-gray-825 shadow-xl rounded p-4 relative"+ className="stats-item-header flex flex-col flex-grow bg-white dark:bg-gray-825 shadow-xl rounded p-4 relative">{/* Header Container */}<div className="w-full flex justify-between">diff --git a/config/runtime.exs b/config/runtime.exsindex 39a5968372..7bc22a736c 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -89,10 +89,6 @@ paddle_auth_code = get_var_from_path_or_env(config_dir, "PADDLE_VENDOR_AUTH_CODEgoogle_cid = get_var_from_path_or_env(config_dir, "GOOGLE_CLIENT_ID")google_secret = get_var_from_path_or_env(config_dir, "GOOGLE_CLIENT_SECRET")slack_hook_url = get_var_from_path_or_env(config_dir, "SLACK_WEBHOOK")-twitter_consumer_key = get_var_from_path_or_env(config_dir, "TWITTER_CONSUMER_KEY")-twitter_consumer_secret = get_var_from_path_or_env(config_dir, "TWITTER_CONSUMER_SECRET")-twitter_token = get_var_from_path_or_env(config_dir, "TWITTER_ACCESS_TOKEN")-twitter_token_secret = get_var_from_path_or_env(config_dir, "TWITTER_ACCESS_TOKEN_SECRET")postmark_api_key = get_var_from_path_or_env(config_dir, "POSTMARK_API_KEY")cron_enabled =@@ -271,12 +267,6 @@ case mailer_adapter doraise "Unknown mailer_adapter; expected SMTPAdapter or PostmarkAdapter"end-config :plausible, :twitter,- consumer_key: twitter_consumer_key,- consumer_secret: twitter_consumer_secret,- token: twitter_token,- token_secret: twitter_token_secret-config :plausible, :custom_domain_server,user: custom_domain_server_user,password: custom_domain_server_password,@@ -296,8 +286,6 @@ if config_env() == :prod && !disable_cron do{"0 * * * *", Plausible.Workers.ScheduleEmailReports},# hourly{"0 * * * *", Plausible.Workers.SendSiteSetupEmails},- # Daily at midnight- {"0 0 * * *", Plausible.Workers.FetchTweets},# Daily at midday{"0 12 * * *", Plausible.Workers.SendCheckStatsEmails},# Every 15 minutes@@ -326,7 +314,6 @@ if config_env() == :prod && !disable_cron doschedule_email_reports: 1,send_email_reports: 1,spike_notifications: 1,- fetch_tweets: 1,check_stats_emails: 1,site_setup_emails: 1,clean_email_verification_codes: 1,diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 8a44d9bda1..71ac9d3bef 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -474,29 +474,11 @@ defmodule Plausible.Stats.Clickhouse do)end- referring_urls =- ClickhouseRepo.all(q)- |> Enum.map(fn ref ->- url = if ref[:name] !== "", do: URI.parse("http://" <> ref[:name]).host- Map.put(ref, :url, url)- end)-- if referrer == "Twitter" do- urls = Enum.map(referring_urls, & &1[:name])-- tweets =- Repo.all(- from t in Plausible.Twitter.Tweet,- where: t.link in ^urls- )- |> Enum.group_by(& &1.link)-- Enum.map(referring_urls, fn url ->- Map.put(url, :tweets, tweets[url[:name]])- end)- else- referring_urls- end+ ClickhouseRepo.all(q)+ |> Enum.map(fn ref ->+ url = if ref[:name] !== "", do: URI.parse("http://" <> ref[:name]).host+ Map.put(ref, :url, url)+ end)enddef referrer_drilldown_for_goal(site, query, referrer) dodiff --git a/lib/plausible/twitter/api.ex b/lib/plausible/twitter/api.exdeleted file mode 100644index bdb13211ef..0000000000--- a/lib/plausible/twitter/api.ex+++ /dev/null@@ -1,24 +0,0 @@-defmodule Plausible.Twitter.Api do- def search(link) do- params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]-- params =- OAuther.sign(- "get",- "https://api.twitter.com/1.1/search/tweets.json",- params,- oauth_credentials()- )-- uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)- response = HTTPoison.get!(uri)-- Jason.decode!(response.body)- |> Map.get("statuses")- end-- defp oauth_credentials() do- Application.get_env(:plausible, :twitter, %{})- |> OAuther.credentials()- end-enddiff --git a/lib/plausible/twitter/tweet.ex b/lib/plausible/twitter/tweet.exdeleted file mode 100644index 0e08fc87fb..0000000000--- a/lib/plausible/twitter/tweet.ex+++ /dev/null@@ -1,34 +0,0 @@-defmodule Plausible.Twitter.Tweet do- use Ecto.Schema- import Ecto.Changeset-- @required_fields [- :link,- :tweet_id,- :author_handle,- :author_name,- :author_image,- :text,- :created- ]-- @derive {Jason.Encoder, only: @required_fields}- schema "tweets" do- field :link, :string-- field :tweet_id, :string- field :author_handle, :string- field :author_name, :string- field :author_image, :string- field :text, :string- field :created, :naive_datetime, null: false-- timestamps()- end-- def changeset(tweet, attrs) do- tweet- |> cast(attrs, @required_fields)- |> validate_required(@required_fields)- end-enddiff --git a/lib/workers/fetch_tweets.ex b/lib/workers/fetch_tweets.exdeleted file mode 100644index 5b9ddbad18..0000000000--- a/lib/workers/fetch_tweets.ex+++ /dev/null@@ -1,57 +0,0 @@-defmodule Plausible.Workers.FetchTweets do- use Plausible.Repo- alias Plausible.Twitter.Tweet- use Oban.Worker, queue: :fetch_tweets-- @impl Oban.Worker- def perform(_job, twitter_api \\ Plausible.Twitter.Api) do- new_links =- Plausible.ClickhouseRepo.all(- from e in Plausible.ClickhouseEvent,- where:- e.timestamp > fragment("(now() - INTERVAL 6 day)") and- e.timestamp < fragment("(now() - INTERVAL 5 day)"),- or_where: e.timestamp > fragment("(now() - INTERVAL 1 day)"),- where: e.referrer_source == "Twitter",- where: e.referrer not in ["t.co", "t.co/"],- distinct: true,- select: e.referrer- )-- for link <- new_links do- results = twitter_api.search(link)-- for tweet <- results do- {:ok, created} =- Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}")-- Tweet.changeset(%Tweet{}, %{- link: link,- tweet_id: tweet["id_str"],- author_handle: tweet["user"]["screen_name"],- author_name: tweet["user"]["name"],- author_image: tweet["user"]["profile_image_url_https"],- text: html_body(tweet),- created: created- })- |> Repo.insert!(on_conflict: :nothing)- end- end-- :ok- end-- def html_body(tweet) do- body =- Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text ->- html = "<a href=\"#{url["url"]}\" target=\"_blank\">#{url["display_url"]}</a>"- String.replace(text, url["url"], html)- end)-- Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text ->- link = "https://twitter.com/#{mention["screen_name"]}"- html = "<a href=\"#{link}\" target=\"_blank\">@#{mention["screen_name"]}</a>"- String.replace(text, "@" <> mention["screen_name"], html)- end)- end-enddiff --git a/priv/repo/migrations/20211202094732_remove_tweets.exs b/priv/repo/migrations/20211202094732_remove_tweets.exsnew file mode 100644index 0000000000..b6ec36a187--- /dev/null+++ b/priv/repo/migrations/20211202094732_remove_tweets.exs@@ -0,0 +1,7 @@+defmodule Plausible.Repo.Migrations.RemoveTweets do+ use Ecto.Migration++ def change do+ drop table(:tweets)+ end+enddiff --git a/test/support/factory.ex b/test/support/factory.exindex 4ffbf7070b..b51cb0a6fe 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -142,17 +142,6 @@ defmodule Plausible.Factory do}end- def tweet_factory do- %Plausible.Twitter.Tweet{- tweet_id: UUID.uuid4(),- author_handle: "author-handle",- author_name: "author-name",- author_image: "pic.twitter.com/author.png",- text: "tweet-text",- created: Timex.now()- }- end-def weekly_report_factory do%Plausible.Site.WeeklyReport{}enddiff --git a/test/workers/fetch_tweets_test.exs b/test/workers/fetch_tweets_test.exsdeleted file mode 100644index b9112c9519..0000000000--- a/test/workers/fetch_tweets_test.exs+++ /dev/null@@ -1,93 +0,0 @@-defmodule Plausible.Workers.FetchTweetsTest do- use Plausible.DataCase- alias Plausible.Workers.FetchTweets- import Double-- test "fetches Twitter referrals from the last day" do- twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end)- FetchTweets.perform(nil, twitter_mock)-- assert_receive({Plausible.Twitter.Api, :search, ["t.co/a-link"]})- end-- test "fetches Twitter referrals from 5-6 days ago" do- twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end)- FetchTweets.perform(nil, twitter_mock)-- assert_receive({Plausible.Twitter.Api, :search, ["t.co/b-link"]})- end-- test "stores twitter results" do- tweet = %{- "full_text" => "a Tweet body",- "id_str" => "the_tweet_id",- "created_at" => "Mon May 06 20:01:29 +0000 2019",- "user" => %{- "screen_name" => "twitter_author",- "name" => "Twitter Author",- "profile_image_url_https" => "https://image.com"- },- "entities" => %{- "user_mentions" => [],- "urls" => []- }- }-- twitter_mock =- stub(Plausible.Twitter.Api, :search, fn- "t.co/a-link" -> [tweet]- _link -> []- end)-- FetchTweets.perform(nil, twitter_mock)-- [found_tweet] = Repo.all(from(t in Plausible.Twitter.Tweet))- assert found_tweet.tweet_id == "the_tweet_id"- assert found_tweet.text == "a Tweet body"- assert found_tweet.author_handle == "twitter_author"- assert found_tweet.author_name == "Twitter Author"- assert found_tweet.author_image == "https://image.com"- assert found_tweet.created == ~N[2019-05-06 20:01:29]- end-- describe "processing tweet entities" do- test "inlines links to the body" do- tweet = %{- "full_text" => "asd https://t.co/somelink",- "entities" => %{- "user_mentions" => [],- "urls" => [- %{- "display_url" => "plausible.io",- "indices" => [4, 17],- "url" => "https://t.co/somelink"- }- ]- }- }-- body = FetchTweets.html_body(tweet)-- assert body == "asd <a href=\"https://t.co/somelink\" target=\"_blank\">plausible.io</a>"- end-- test "inlines user mentions to the body" do- tweet = %{- "full_text" => "asd @hello",- "entities" => %{- "user_mentions" => [- %{- "screen_name" => "hello",- "id_str" => "123123"- }- ],- "urls" => []- }- }-- body = FetchTweets.html_body(tweet)-- assert body == "asd <a href=\"https://twitter.com/hello\" target=\"_blank\">@hello</a>"- end- end-end