Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(improvement) Migrate from postmark to AWS SES (#229)

Andrii Romasiunfb21b7c47062
diff --git a/.env.example b/.env.example
index 36b38b30c..f454919b7 100644
--- a/.env.example
+++ b/.env.example
@@ -33,7 +33,6 @@ API_ORIGINS=
# EMAIL SERVICE
SMTP_HOST=localhost
-SMTP_BROADCAST_HOST=localhost
SMTP_PORT=1025
SMTP_USER=smtp_user_name
SMTP_PASSWORD=smtp_password
@@ -89,3 +88,6 @@ CLOUDFLARE_PROXY_ENABLED=false
# Paypal (used for payouts to referrers)
PAYPAL_CLIENT_ID=
PAYPAL_CLIENT_SECRET=
+
+# Generic
+EMAIL_ACTION_ENCRYPTION_KEY=
diff --git a/Dockerfile.prod b/Dockerfile.prod
index 37e9de8c8..d5ce2a423 100644
--- a/Dockerfile.prod
+++ b/Dockerfile.prod
@@ -26,7 +26,6 @@ ENV TZ=UTC \
CLICKHOUSE_DATABASE=analytics \
API_ORIGINS= \
SMTP_HOST=localhost \
- SMTP_BROADCAST_HOST=localhost \
SMTP_PORT=1025 \
SMTP_USER=smtp_user_name \
SMTP_PASSWORD=smtp_password \
diff --git a/apps/production/src/app.module.ts b/apps/production/src/app.module.ts
index a688b9b44..a8572d89b 100644
--- a/apps/production/src/app.module.ts
+++ b/apps/production/src/app.module.ts
@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'
import { ConfigModule } from '@nestjs/config'
import { ScheduleModule } from '@nestjs/schedule'
import { NestjsFormDataModule } from 'nestjs-form-data'
+import { MailerModule as NodeMailerModule } from '@nestjs-modules/mailer'
import { I18nModule } from 'nestjs-i18n'
import { UserModule } from './user/user.module'
@@ -30,6 +31,22 @@ const modules = [
expandVariables: true,
isGlobal: true,
}),
+ NodeMailerModule.forRoot({
+ transport: {
+ sendingRate: 14,
+ // pool: true, // if true - set up pooled connections against a SMTP server
+ host: process.env.SMTP_HOST,
+ port: Number(process.env.SMTP_PORT),
+ secure: true, // if false - upgrade later with STARTTLS
+ auth: {
+ user: process.env.SMTP_USER,
+ pass: process.env.SMTP_PASSWORD,
+ },
+ },
+ defaults: {
+ from: `"Swetrix" <${process.env.FROM_EMAIL}>`, // outgoing email ID
+ },
+ }),
TypeOrmModule.forRoot({
type: 'mysql',
host: process.env.MYSQL_HOST,
diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.ts
index 306826076..2487caaff 100644
--- a/apps/production/src/common/constants.ts
+++ b/apps/production/src/common/constants.ts
@@ -7,8 +7,12 @@ import * as _toNumber from 'lodash/toNumber'
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('dotenv').config()
-const { CLICKHOUSE_DATABASE, PAYPAL_CLIENT_ID, PAYPAL_CLIENT_SECRET } =
- process.env
+const {
+ CLICKHOUSE_DATABASE,
+ PAYPAL_CLIENT_ID,
+ PAYPAL_CLIENT_SECRET,
+ EMAIL_ACTION_ENCRYPTION_KEY,
+} = process.env
const redis = new Redis(
_toNumber(process.env.REDIS_PORT),
@@ -211,6 +215,7 @@ export {
REDIS_LOG_PERF_CACHE_KEY,
CAPTCHA_SALT,
CAPTCHA_ENCRYPTION_KEY,
+ EMAIL_ACTION_ENCRYPTION_KEY,
isDevelopment,
getRedisCaptchaKey,
CAPTCHA_COOKIE_KEY,
diff --git a/apps/production/src/common/templates/en/customer_sign-up.html b/apps/production/src/common/templates/en/customer_sign-up.html
index 432896a2f..23054afa4 100644
--- a/apps/production/src/common/templates/en/customer_sign-up.html
+++ b/apps/production/src/common/templates/en/customer_sign-up.html
@@ -1,7 +1,6 @@
Hello and welcome to Swetrix! We're excited to have you with us.<br/>
Swetrix is an ethical and powerful web analytics platform that respects your privacy and data.<br/><br/>
-To get started, please verify your email address by clicking the link below:<br/>
-<a href="{{url}}">Verify registration</a><br/><br/>
+To get started, please <a href="{{url}}">click here</a> verify your email address.<br/><br/>
If you’re having trouble with the link above, copy and paste the URL below directly into your web browser:<br/>
{{url}}<br/><br/>
If you did not request this email, feel free to ignore it :)
diff --git a/apps/production/src/common/templates/en/project-report.html b/apps/production/src/common/templates/en/project-report.html
index 2aa3c2915..13e7a4f83 100644
--- a/apps/production/src/common/templates/en/project-report.html
+++ b/apps/production/src/common/templates/en/project-report.html
@@ -97,8 +97,8 @@
<br /><br />
- You can manage email reports preferences in your <a href="https://swetrix.com/settings"
- target="_blank">account settings</a>.
+ Do not want to receive these emails? <a href="{{data.unsubscribeUrl}}"
+ target="_blank">Click here</a> to unsubscribe.
</span>
</font>
<div style="height: 30px; line-height: 30px; font-size: 28px;">&nbsp;</div>
diff --git a/apps/production/src/mailer/mailer.service.ts b/apps/production/src/mailer/mailer.service.ts
index 63f655e70..722d52e73 100644
--- a/apps/production/src/mailer/mailer.service.ts
+++ b/apps/production/src/mailer/mailer.service.ts
@@ -1,7 +1,7 @@
import fs = require('fs')
import path = require('path')
+import { MailerService as NodeMailerService } from '@nestjs-modules/mailer'
import { Injectable } from '@nestjs/common'
-import * as postmark from 'postmark'
import handlebars from 'handlebars'
import { LetterTemplate } from './letter'
import { AppLoggerService } from '../logger/logger.service'
@@ -104,17 +104,19 @@ const metaInfoJson = {
},
[LetterTemplate.UsageOverLimit]: {
subject: {
- en: () => 'You have exceeded your Swetrix subscription plan',
+ en: () =>
+ '[ACTION REQUIRED] You have exceeded your Swetrix subscription plan',
},
},
[LetterTemplate.DashboardLockedExceedingLimits]: {
subject: {
- en: () => 'Your Swetrix dashboard has been locked',
+ en: () => '[ACTION REQUIRED] Your Swetrix dashboard has been locked',
},
},
[LetterTemplate.DashboardLockedPaymentFailure]: {
subject: {
- en: () => 'Your Swetrix dashboard has been locked due to a payment issue',
+ en: () =>
+ '[ACTION REQUIRED] Your Swetrix dashboard has been locked due to a payment issue',
},
},
}
@@ -143,31 +145,30 @@ handlebars.registerHelper('greater', function greater(v1, v2, options) {
return options.inverse(this)
})
-const mailClient = new postmark.ServerClient(process.env.SMTP_PASSWORD)
-
@Injectable()
export class MailerService {
- constructor(private readonly logger: AppLoggerService) {}
+ constructor(
+ private readonly logger: AppLoggerService,
+ private readonly nodeMailerService: NodeMailerService,
+ ) {}
async sendEmail(
email: string,
templateName: LetterTemplate,
params: Params = null,
- messageStream: 'broadcast' | 'outbound' = 'outbound',
): Promise<void> {
try {
const templatePath = `${TEMPLATES_PATH}/en/${templateName}.html`
const letter = fs.readFileSync(templatePath, { encoding: 'utf-8' })
const subject = metaInfoJson[templateName].subject.en(params)
const template = handlebars.compile(letter)
- const htmlToSend = template(params)
+ const html = template(params)
- const message: postmark.Models.Message = {
- From: process.env.FROM_EMAIL,
- To: email,
- Subject: subject,
- HtmlBody: htmlToSend,
- MessageStream: messageStream,
+ const message = {
+ from: process.env.FROM_EMAIL,
+ to: email,
+ subject,
+ html,
}
if (process.env.SMTP_MOCK) {
@@ -180,10 +181,10 @@ export class MailerService {
true,
)
} else {
- await mailClient.sendEmail(message)
+ await this.nodeMailerService.sendMail(message)
}
- } catch (error) {
- console.error(`[ERROR][MAILER] ${error}`)
+ } catch (reason) {
+ this.logger.error(reason, 'sendEmail', true)
}
}
}
diff --git a/apps/production/src/main.ts b/apps/production/src/main.ts
index b1f28e89a..2f4d1519c 100644
--- a/apps/production/src/main.ts
+++ b/apps/production/src/main.ts
@@ -76,6 +76,9 @@ async function bootstrap() {
app.use('/webhook', bodyParser.raw({ type: 'application/json' }))
+ app.use('/webhook/sns', bodyParser.raw())
+ app.use('/webhook/sns', bodyParser.text())
+
if (process.env.NODE_ENV === 'production') {
const bot = app.get(getBotToken())
app.use(bot.webhookCallback(process.env.TELEGRAM_WEBHOOK_PATH))
diff --git a/apps/production/src/project/project.controller.ts b/apps/production/src/project/project.controller.ts
index fe8eb66b5..8189f289c 100644
--- a/apps/production/src/project/project.controller.ts
+++ b/apps/production/src/project/project.controller.ts
@@ -1602,6 +1602,32 @@ export class ProjectController {
return _omit(project, ['passwordHash'])
}
+ // Used to unsubscribe from email reports for 3rd party users (i.e. project-subscriber.entity.ts)
+ @Get('/unsubscribe/:token')
+ @Public()
+ @ApiResponse({ status: 204 })
+ async unsubscribeFromEmailReports(
+ @Param('token') token: string,
+ ): Promise<void> {
+ this.logger.log({ token }, 'GET /project/unsubscribe/:token')
+
+ let subscriberId
+
+ try {
+ subscriberId = this.projectService.decryptUnsubscribeKey(token)
+ } catch {
+ throw new NotFoundException('Unsubscribe token is invalid')
+ }
+
+ const subscriber = await this.projectService.getSubscriberById(subscriberId)
+
+ if (!subscriber) {
+ throw new NotFoundException('Unsubscribe token is invalid')
+ }
+
+ await this.projectService.removeSubscriberById(subscriberId)
+ }
+
@Get('/ogimage/:id')
@HttpCode(200)
@Header('Content-Type', 'image/jpeg')
diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.ts
index 96c43100c..5a8823a62 100644
--- a/apps/production/src/project/project.service.ts
+++ b/apps/production/src/project/project.service.ts
@@ -12,6 +12,7 @@ import { Repository } from 'typeorm'
import { customAlphabet } from 'nanoid'
import handlebars from 'handlebars'
import puppeteer from 'puppeteer'
+import * as CryptoJS from 'crypto-js'
import * as _isEmpty from 'lodash/isEmpty'
import * as _size from 'lodash/size'
import * as _join from 'lodash/join'
@@ -51,6 +52,7 @@ import {
getRedisUserUsageInfoKey,
redisUserUsageinfoCacheTimeout,
TRAFFIC_COLUMNS,
+ EMAIL_ACTION_ENCRYPTION_KEY,
} from '../common/constants'
import { IUsageInfoRedis } from '../user/interfaces'
import { ProjectSubscriber, Funnel } from './entity'
@@ -922,12 +924,22 @@ export class ProjectService {
})
}
+ async findOneSubscriber(where: Record<string, unknown>) {
+ return this.projectSubscriberRepository.findOne({ where })
+ }
+
async getSubscriber(projectId: string, subscriberId: string) {
return this.projectSubscriberRepository.findOne({
where: { id: subscriberId, projectId },
})
}
+ async getSubscriberById(id: string) {
+ return this.projectSubscriberRepository.findOne({
+ where: { id },
+ })
+ }
+
async confirmSubscriber(
projectId: string,
subscriberId: string,
@@ -997,6 +1009,12 @@ export class ProjectService {
})
}
+ async removeSubscriberById(id: string) {
+ await this.projectSubscriberRepository.delete({
+ id,
+ })
+ }
+
async getSubscribersForReports(reportFrequency: ReportFrequency) {
return this.projectSubscriberRepository.find({
relations: ['project'],
@@ -1018,6 +1036,23 @@ export class ProjectService {
})
}
+ createUnsubscribeKey(subscriberId: string): string {
+ return encodeURIComponent(
+ CryptoJS.Rabbit.encrypt(
+ subscriberId,
+ EMAIL_ACTION_ENCRYPTION_KEY,
+ ).toString(),
+ )
+ }
+
+ decryptUnsubscribeKey(token: string): string {
+ const bytes = CryptoJS.Rabbit.decrypt(
+ decodeURIComponent(token),
+ EMAIL_ACTION_ENCRYPTION_KEY,
+ )
+ return bytes.toString(CryptoJS.enc.Utf8)
+ }
+
async transferProject(
projectId: string,
name: string,
diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.ts
index 209d6ab69..357537efa 100644
--- a/apps/production/src/task-manager/task-manager.service.ts
+++ b/apps/production/src/task-manager/task-manager.service.ts
@@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common'
import { Cron, CronExpression } from '@nestjs/schedule'
import { IsNull, LessThan, In, Not, Between, MoreThan, Like } from 'typeorm'
+import { ConfigService } from '@nestjs/config'
import * as Paypal from '@paypal/payouts-sdk'
import * as bcrypt from 'bcrypt'
import * as dayjs from 'dayjs'
@@ -231,6 +232,24 @@ const getUsersThatExceedContinuously = (
return exceedingUsers as User & { usage: any[] }[]
}
+const EMAIL_REPORTS_MAP = {
+ [ReportFrequency.WEEKLY]: {
+ type: 'w',
+ dayjsParams: [1, 'w'],
+ analyticsParam: '7d',
+ },
+ [ReportFrequency.MONTHLY]: {
+ type: 'M',
+ dayjsParams: [1, 'M'],
+ analyticsParam: '4w',
+ },
+ [ReportFrequency.QUARTERLY]: {
+ type: 'M',
+ dayjsParams: [3, 'M'],
+ analyticsParam: '3M',
+ },
+}
+
@Injectable()
export class TaskManagerService {
constructor(
@@ -244,8 +263,152 @@ export class TaskManagerService {
private readonly logger: AppLoggerService,
private readonly telegramService: TelegramService,
private readonly payoutsService: PayoutsService,
+ private readonly configService: ConfigService,
) {}
+ generateUnsubscribeUrl(
+ id: string,
+ type: '3rdparty' | 'user-reports',
+ ): string {
+ if (type === '3rdparty') {
+ const token = this.projectService.createUnsubscribeKey(id)
+ return `${this.configService.get(
+ 'CLIENT_URL',
+ )}/3rd-party-unsubscribe/${token}`
+ }
+
+ const token = this.userService.createUnsubscribeKey(id)
+ return `${this.configService.get(
+ 'CLIENT_URL',
+ )}/reports-unsubscribe/${token}`
+ }
+
+ async handleUserReports(
+ reportFrequency:
+ | ReportFrequency.WEEKLY
+ | ReportFrequency.MONTHLY
+ | ReportFrequency.QUARTERLY,
+ ): Promise<void> {
+ const params = EMAIL_REPORTS_MAP[reportFrequency]
+
+ const users = await this.userService.find({
+ where: {
+ reportFrequency,
+ planCode: Not(PlanCode.none),
+ dashboardBlockReason: IsNull(),
+ },
+ relations: ['projects'],
+ select: ['id', 'email'],
+ })
+ const now = dayjs.utc().format('DD.MM.YYYY')
+ const timeAgo = dayjs
+ .utc()
+ // @ts-ignore
+ .subtract(...params.dayjsParams)
+ .format('DD.MM.YYYY')
+ const date = `${timeAgo} - ${now}`
+ const tip = getRandomTip()
+
+ const promises = _map(users, async user => {
+ const { id, email, projects } = user
+
+ if (_isEmpty(projects) || _isNull(projects)) {
+ return
+ }
+
+ const unsubscribeUrl = this.generateUnsubscribeUrl(id, 'user-reports')
+
+ const ids = _map(projects, p => p.id)
+ const data = this.analyticsService.convertSummaryToObsoleteFormat(
+ await this.analyticsService.getAnalyticsSummary(
+ ids,
+ params.analyticsParam,
+ ),
+ )
+
+ const result = {
+ type: params.type,
+ date,
+ projects: _map(ids, (pid, index) => ({
+ data: data[pid],
+ name: projects[index].name,
+ })),
+ tip,
+ unsubscribeUrl,
+ }
+
+ await this.mailerService.sendEmail(
+ email,
+ LetterTemplate.ProjectReport,
+ result,
+ )
+ })
+
+ await Promise.allSettled(promises).catch(reason => {
+ this.logger.error(
+ `[CRON WORKER](handleUserReports) Frequency: ${reportFrequency}; Error occured: ${reason}`,
+ )
+ })
+ }
+
+ async handleSubscriberReports(
+ reportFrequency:
+ | ReportFrequency.WEEKLY
+ | ReportFrequency.MONTHLY
+ | ReportFrequency.QUARTERLY,
+ ): Promise<void> {
+ const params = EMAIL_REPORTS_MAP[reportFrequency]
+
+ const subscribers =
+ await this.projectService.getSubscribersForReports(reportFrequency)
+ const now = dayjs.utc().format('DD.MM.YYYY')
+ const timeAgo = dayjs
+ .utc()
+ // @ts-ignore
+ .subtract(...params.dayjsParams)
+ .format('DD.MM.YYYY')
+ const date = `${timeAgo} - ${now}`
+ const tip = getRandomTip()
+
+ const promises = _map(subscribers, async subscriber => {
+ const { id, email } = subscriber
+ const projects = await this.projectService.getSubscriberProjects(id)
+
+ const unsubscribeUrl = this.generateUnsubscribeUrl(id, '3rdparty')
+
+ const ids = projects.map(project => project.id)
+ const data = this.analyticsService.convertSummaryToObsoleteFormat(
+ await this.analyticsService.getAnalyticsSummary(
+ ids,
+ params.analyticsParam,
+ ),
+ )
+
+ const result = {
+ type: params.type,
+ date,
+ projects: _map(ids, (pid, index) => ({
+ data: data[pid],
+ name: projects[index].name,
+ })),
+ tip,
+ unsubscribeUrl,
+ }
+
+ await this.mailerService.sendEmail(
+ email,
+ LetterTemplate.ProjectReport,
+ result,
+ )
+ })
+
+ await Promise.allSettled(promises).catch(reason => {
+ this.logger.error(
+ `[CRON WORKER](handleSubscriberReports) Frequency: ${reportFrequency}; Error: ${reason}`,
+ )
+ })
+ }
+
@Cron(CronExpression.EVERY_MINUTE)
async saveLogData(): Promise<void> {
const data = await redis.lrange(REDIS_LOG_DATA_CACHE_KEY, 0, -1)
@@ -547,7 +710,6 @@ export class TaskManagerService {
email,
LetterTemplate.TierWarning,
emailParams,
- 'broadcast',
)
await this.userService.update(id, {
evWarningSentOn: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
@@ -595,298 +757,37 @@ export class TaskManagerService {
// EVERY SUNDAY AT 2:30 AM
@Cron('30 02 * * 0')
async weeklyReportsHandler(): Promise<void> {
- const users = await this.userService.find({
- where: {
- reportFrequency: ReportFrequency.WEEKLY,
- planCode: Not(PlanCode.none),
- dashboardBlockReason: IsNull(),
- },
- relations: ['projects'],
- select: ['email'],
- })
- const now = dayjs.utc().format('DD.MM.YYYY')
- const weekAgo = dayjs.utc().subtract(1, 'w').format('DD.MM.YYYY')
- const date = `${weekAgo} - ${now}`
- const tip = getRandomTip()
-
- const promises = _map(users, async user => {
- const { email, projects } = user
-
- if (_isEmpty(projects) || _isNull(projects)) {
- return
- }
-
- const ids = _map(projects, p => p.id)
- const data = this.analyticsService.convertSummaryToObsoleteFormat(
- await this.analyticsService.getAnalyticsSummary(ids, '7d'),
- )
-
- const result = {
- type: 'w', // week
- date,
- projects: _map(ids, (pid, index) => ({
- data: data[pid],
- name: projects[index].name,
- })),
- tip,
- }
-
- await this.mailerService.sendEmail(
- email,
- LetterTemplate.ProjectReport,
- result,
- 'broadcast',
- )
- })
-
- await Promise.allSettled(promises).catch(reason => {
- this.logger.error(
- `[CRON WORKER](weeklyReportsHandler) Error occured: ${reason}`,
- )
- })
+ await this.handleUserReports(ReportFrequency.WEEKLY)
}
// ON THE FIRST DAY OF EVERY MONTH AT 2 AM
@Cron('0 02 1 * *')
async monthlyReportsHandler(): Promise<void> {
- const users = await this.userService.find({
- where: {
- reportFrequency: ReportFrequency.MONTHLY,
- planCode: Not(PlanCode.none),
- dashboardBlockReason: IsNull(),
- },
- relations: ['projects'],
- select: ['email'],
- })
- const now = dayjs.utc().format('DD.MM.YYYY')
- const weekAgo = dayjs.utc().subtract(1, 'M').format('DD.MM.YYYY')
- const date = `${weekAgo} - ${now}`
- const tip = getRandomTip()
-
- const promises = _map(users, async user => {
- const { email, projects } = user
-
- if (_isEmpty(projects) || _isNull(projects)) {
- return
- }
-
- const ids = _map(projects, p => p.id)
- const data = this.analyticsService.convertSummaryToObsoleteFormat(
- await this.analyticsService.getAnalyticsSummary(ids, '4w'),
- )
-
- const result = {
- type: 'M', // month
- date,
- projects: _map(ids, (pid, index) => ({
- data: data[pid],
- name: projects[index].name,
- })),
- tip,
- }
-
- await this.mailerService.sendEmail(
- email,
- LetterTemplate.ProjectReport,
- result,
- 'broadcast',
- )
- })
-
- await Promise.allSettled(promises).catch(reason => {
- this.logger.error(
- `[CRON WORKER](monthlyReportsHandler) Error occured: ${reason}`,
- )
- })
+ await this.handleUserReports(ReportFrequency.MONTHLY)
}
@Cron(CronExpression.EVERY_QUARTER)
async quarterlyReportsHandler(): Promise<void> {
- const users = await this.userService.find({
- where: {
- reportFrequency: ReportFrequency.QUARTERLY,
- planCode: Not(PlanCode.none),
- dashboardBlockReason: IsNull(),
- },
- relations: ['projects'],
- select: ['email'],
- })
- const now = dayjs.utc().format('DD.MM.YYYY')
- const quarterAgo = dayjs.utc().subtract(3, 'M').format('DD.MM.YYYY')
- const date = `${quarterAgo} - ${now}`
- const tip = getRandomTip()
-
- const promises = _map(users, async user => {
- const { email, projects } = user
-
- if (_isEmpty(projects) || _isNull(projects)) {
- return
- }
-
- const ids = _map(projects, p => p.id)
- const data = this.analyticsService.convertSummaryToObsoleteFormat(
- await this.analyticsService.getAnalyticsSummary(ids, '3M'),
- )
-
- const result = {
- type: 'Q', // quarterly
- date,
- projects: _map(ids, (pid, index) => ({
- data: data[pid],
- name: projects[index].name,
- })),
- tip,
- }
-
- await this.mailerService.sendEmail(
- email,
- LetterTemplate.ProjectReport,
- result,
- 'broadcast',
- )
- })
-
- await Promise.allSettled(promises).catch(reason => {
- this.logger.error(
- `[CRON WORKER](quarterlyReportsHandler) Error occured: ${reason}`,
- )
- })
+ await this.handleUserReports(ReportFrequency.QUARTERLY)
}
// EMAIL REPORTS, BUT FOR MULTIPLE PROJECT SUBSCRIBERS
@Cron(CronExpression.EVERY_QUARTER)
async handleQuarterlyReports(): Promise<void> {
- const subscribers = await this.projectService.getSubscribersForReports(
- ReportFrequency.QUARTERLY,
- )
- const now = dayjs.utc().format('DD.MM.YYYY')
- const quarterAgo = dayjs.utc().subtract(3, 'M').format('DD.MM.YYYY')
- const date = `${quarterAgo} - ${now}`
- const tip = getRandomTip()
-
- const promises = _map(subscribers, async subscriber => {
- const { id, email } = subscriber
- const projects = await this.projectService.getSubscriberProjects(id)
-
- const ids = projects.map(project => project.id)
- const data = this.analyticsService.convertSummaryToObsoleteFormat(
- await this.analyticsService.getAnalyticsSummary(ids, '3M'),
- )
-
- const result = {
- type: 'Q', // quarter
- date,
- projects: _map(ids, (pid, index) => ({
- data: data[pid],
- name: projects[index].name,
- })),
- tip,
- }
-
- await this.mailerService.sendEmail(
- email,
- LetterTemplate.ProjectReport,
- result,
- 'broadcast',
- )
- })
-
- await Promise.allSettled(promises).catch(reason => {
- this.logger.error(
- `[CRON WORKER](handleQuarterlyReports) Error occured: ${reason}`,
- )
- })
+ await this.handleSubscriberReports(ReportFrequency.QUARTERLY)
}
// ON THE FIRST DAY OF EVERY MONTH AT 3 AM
@Cron('0 03 1 * *')
async handleMonthlyReports(): Promise<void> {
- const subscribers = await this.projectService.getSubscribersForReports(
- ReportFrequency.MONTHLY,
- )
- const now = dayjs.utc().format('DD.MM.YYYY')
- const weekAgo = dayjs.utc().subtract(1, 'M').format('DD.MM.YYYY')
- const date = `${weekAgo} - ${now}`
- const tip = getRandomTip()
-
- const promises = _map(subscribers, async subscriber => {
- const { id, email } = subscriber
- const projects = await this.projectService.getSubscriberProjects(id)
-
- const ids = projects.map(project => project.id)
- const data = this.analyticsService.convertSummaryToObsoleteFormat(
- await this.analyticsService.getAnalyticsSummary(ids, '4w'),
- )
-
- const result = {
- type: 'M', // month
- date,
- projects: _map(ids, (pid, index) => ({
- data: data[pid],
- name: projects[index].name,
- })),
- tip,
- }
-
- await this.mailerService.sendEmail(
- email,
- LetterTemplate.ProjectReport,
- result,
- 'broadcast',
- )
- })
-
- await Promise.allSettled(promises).catch(reason => {
- this.logger.error(
- `[CRON WORKER](handleMonthlyReports) Error occured: ${reason}`,
- )
- })
+ await this.handleSubscriberReports(ReportFrequency.MONTHLY)
}
// EVERY SUNDAY AT 3 AM
@Cron('0 03 * * 0')
async handleWeeklyReports(): Promise<void> {
- const subscribers = await this.projectService.getSubscribersForReports(
- ReportFrequency.WEEKLY,
- )
- const now = dayjs.utc().format('DD.MM.YYYY')
- const weekAgo = dayjs.utc().subtract(1, 'w').format('DD.MM.YYYY')
- const date = `${weekAgo} - ${now}`
- const tip = getRandomTip()
-
- const promises = _map(subscribers, async subscriber => {
- const { id, email } = subscriber
- const projects = await this.projectService.getSubscriberProjects(id)
-
- const ids = projects.map(project => project.id)
- const data = this.analyticsService.convertSummaryToObsoleteFormat(
- await this.analyticsService.getAnalyticsSummary(ids, '7d'),
- )
-
- const result = {
- type: 'w', // week
- date,
- projects: _map(ids, (pid, index) => ({
- data: data[pid],
- name: projects[index].name,
- })),
- tip,
- }
-
- await this.mailerService.sendEmail(
- email,
- LetterTemplate.ProjectReport,
- result,
- 'broadcast',
- )
- })
-
- await Promise.allSettled(promises).catch(reason => {
- this.logger.error(
- `[CRON WORKER](handleWeeklyReports) Error occured: ${reason}`,
- )
- })
+ await this.handleSubscriberReports(ReportFrequency.WEEKLY)
}
@Cron(CronExpression.EVERY_10_MINUTES)
diff --git a/apps/production/src/user/user.controller.ts b/apps/production/src/user/user.controller.ts
index 53a157a78..fa9f98d71 100644
--- a/apps/production/src/user/user.controller.ts
+++ b/apps/production/src/user/user.controller.ts
@@ -16,6 +16,7 @@ import {
Headers,
Ip,
Patch,
+ NotFoundException,
} from '@nestjs/common'
import { Request } from 'express'
import { ApiTags, ApiQuery, ApiResponse, ApiBearerAuth } from '@nestjs/swagger'
@@ -72,6 +73,7 @@ import {
generateRefCode,
} from '../common/utils'
import { IUsageInfo, IMetaInfo } from './interfaces'
+import { ReportFrequency } from '../project/enums'
dayjs.extend(utc)
@@ -899,4 +901,34 @@ export class UserController {
return this.userService.previewSubscription(id, planId)
}
+
+ // Used to unsubscribe from email reports
+ @Get('/unsubscribe/:token')
+ @Public()
+ @ApiResponse({ status: 204 })
+ async unsubscribeFromEmailReports(
+ @Param('token') token: string,
+ ): Promise<void> {
+ this.logger.log({ token }, 'GET /project/unsubscribe/:token')
+
+ let userId
+
+ try {
+ userId = this.userService.decryptUnsubscribeKey(token)
+ } catch {
+ throw new NotFoundException('Unsubscribe token is invalid')
+ }
+
+ const user = await this.userService.findOneWhere({
+ id: userId,
+ })
+
+ if (!user) {
+ throw new NotFoundException('Unsubscribe token is invalid')
+ }
+
+ await this.userService.update(userId, {
+ reportFrequency: ReportFrequency.NEVER,
+ })
+ }
}
diff --git a/apps/production/src/user/user.service.ts b/apps/production/src/user/user.service.ts
index 2c65c5700..ea1f92a3c 100644
--- a/apps/production/src/user/user.service.ts
+++ b/apps/production/src/user/user.service.ts
@@ -6,6 +6,7 @@ import {
import { InjectRepository } from '@nestjs/typeorm'
import { In, Repository } from 'typeorm'
import axios from 'axios'
+import * as CryptoJS from 'crypto-js'
import * as dayjs from 'dayjs'
import * as _isEmpty from 'lodash/isEmpty'
import * as _size from 'lodash/size'
@@ -28,6 +29,7 @@ import { RefreshToken } from './entities/refresh-token.entity'
import { DeleteFeedback } from './entities/delete-feedback.entity'
import { UserGoogleDTO } from './dto/user-google.dto'
import { UserGithubDTO } from './dto/user-github.dto'
+import { EMAIL_ACTION_ENCRYPTION_KEY } from '../common/constants'
const EUR = {
symbol: '€',
@@ -508,6 +510,20 @@ export class UserService {
)
}
+ createUnsubscribeKey(userId: string): string {
+ return encodeURIComponent(
+ CryptoJS.Rabbit.encrypt(userId, EMAIL_ACTION_ENCRYPTION_KEY).toString(),
+ )
+ }
+
+ decryptUnsubscribeKey(token: string): string {
+ const bytes = CryptoJS.Rabbit.decrypt(
+ decodeURIComponent(token),
+ EMAIL_ACTION_ENCRYPTION_KEY,
+ )
+ return bytes.toString(CryptoJS.enc.Utf8)
+ }
+
async getReferralsList(user: User): Promise<Partial<User>[]> {
return this.usersRepository
.createQueryBuilder('user')
diff --git a/apps/production/src/webhook/webhook.controller.ts b/apps/production/src/webhook/webhook.controller.ts
index bf8b408f0..2c82c22ae 100644
--- a/apps/production/src/webhook/webhook.controller.ts
+++ b/apps/production/src/webhook/webhook.controller.ts
@@ -39,6 +39,44 @@ export class WebhookController {
private readonly mailerService: MailerService,
) {}
+ // AWS SNS webhook
+ @Post('/sns')
+ @HttpCode(200)
+ async snsWebhook(@Body() body): Promise<any> {
+ const json = typeof body === 'string' ? JSON.parse(body) : body
+
+ await this.webhookService.verifySNSRequest(json)
+
+ const { Type, Message } = json
+
+ if (Type !== 'Notification') {
+ return
+ }
+
+ const message = typeof Message === 'string' ? JSON.parse(Message) : Message
+
+ switch (message.eventType) {
+ case 'Bounce': {
+ const { emailAddress } = message.bounce.bouncedRecipients[0]
+
+ await this.webhookService.unsubscribeByEmail(emailAddress)
+
+ break
+ }
+
+ case 'Complaint': {
+ const { emailAddress } = message.complaint.complainedRecipients[0]
+
+ await this.webhookService.unsubscribeByEmail(emailAddress)
+
+ break
+ }
+
+ default:
+ }
+ }
+
+ // Paddle - payment processor webhook
@Post('/paddle')
@HttpCode(200)
async paddleWebhook(
diff --git a/apps/production/src/webhook/webhook.service.ts b/apps/production/src/webhook/webhook.service.ts
index 747b4c226..b4dcfd697 100644
--- a/apps/production/src/webhook/webhook.service.ts
+++ b/apps/production/src/webhook/webhook.service.ts
@@ -1,5 +1,6 @@
/* eslint-disable no-param-reassign, no-prototype-builtins */
import * as crypto from 'crypto'
+import * as Validator from 'sns-payload-validator'
import {
Injectable,
BadRequestException,
@@ -11,8 +12,11 @@ import * as _keys from 'lodash/keys'
import * as _isArray from 'lodash/isArray'
import { AppLoggerService } from '../logger/logger.service'
import { PayoutsService } from '../payouts/payouts.service'
+import { ProjectService } from '../project/project.service'
+import { UserService } from '../user/user.service'
import { PayoutStatus } from '../payouts/entities/payouts.entity'
import { User } from '../user/entities/user.entity'
+import { ReportFrequency } from '../project/enums'
const PADDLE_PUB_KEY = `-----BEGIN PUBLIC KEY-----
MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAqFcHslKkXcJlTYg4FL6j
@@ -40,11 +44,15 @@ const paddleWhitelistIPs = [
'3.208.120.145',
]
+const validator = new Validator()
+
@Injectable()
export class WebhookService {
constructor(
private readonly logger: AppLoggerService,
private readonly payoutsService: PayoutsService,
+ private readonly userService: UserService,
+ private readonly projectService: ProjectService,
) {}
public ksort(obj: Record<string, unknown>): Record<string, unknown> {
@@ -125,4 +133,43 @@ export class WebhookService {
},
)
}
+
+ async verifySNSRequest(payload: any): Promise<void> {
+ try {
+ await validator.validate(payload)
+ } catch {
+ throw new BadRequestException('Webhook signature verification failed')
+ }
+ }
+
+ async unsubscribeByEmail(email: string): Promise<void> {
+ const user = await this.userService.findOneWhere({
+ email,
+ })
+
+ // checking if the bounce originates from admin email reports
+ if (user) {
+ await this.userService.update(user.id, {
+ reportFrequency: ReportFrequency.NEVER,
+ })
+ return
+ }
+
+ // checking if the bounce originates from project subscriber email reports
+ const subscribtion = await this.projectService.findOneSubscriber({
+ email,
+ })
+
+ if (subscribtion) {
+ await this.projectService.removeSubscriberById(subscribtion.id)
+ }
+
+ // if it's some other kind of bounce, we can ignore it because it originates from other transactional emails
+ // but I'll log it here for now
+ this.logger.log(
+ `Received an email notification, but it originates from other transactional emails. Email: ${email}`,
+ 'POST /webhook/ses',
+ true,
+ )
+ }
}
diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml
index 64877bc7b..ee25b6d28 100644
--- a/docker-compose-prod.yml
+++ b/docker-compose-prod.yml
@@ -29,7 +29,6 @@ services:
- CLICKHOUSE_HOST=http://clickhouse
- API_ORIGINS=
- SMTP_HOST=localhost
- - SMTP_BROADCAST_HOST=localhost
- SMTP_PORT=1025
- SMTP_USER=smtp_user_name
- SMTP_PASSWORD=smtp_password
diff --git a/package.json b/package.json
index 8fa8e6951..b7411ffdd 100644
--- a/package.json
+++ b/package.json
@@ -27,6 +27,7 @@
},
"dependencies": {
"@faker-js/faker": "^8.3.1",
+ "@nestjs-modules/mailer": "^1.11.2",
"@nestjs/axios": "^3.0.1",
"@nestjs/common": "^10.2.10",
"@nestjs/config": "^3.1.1",
@@ -71,17 +72,18 @@
"nestjs-form-data": "^1.9.4",
"nestjs-i18n": "^10.4.0",
"nestjs-telegraf": "^2.7.0",
+ "nodemailer": "^6.9.13",
"otplib": "^12.0.1",
"passport": "^0.7.0",
"passport-headerapikey": "^1.2.2",
"passport-jwt": "^4.0.1",
"php-serialize": "^4.1.1",
- "postmark": "^4.0.2",
"puppeteer": "^21.6.0",
"randomstring": "^1.3.0",
"reflect-metadata": "^0.1.14",
"rimraf": "^3.0.2",
"rxjs": "^7.8.1",
+ "sns-payload-validator": "^2.1.0",
"svg-captcha": "^1.4.0",
"telegraf": "^4.15.3",
"typeorm": "^0.2.45",
@@ -98,6 +100,7 @@
"@types/jest": "^29.5.11",
"@types/multer": "^1.4.11",
"@types/node": "^20.10.4",
+ "@types/nodemailer": "^6.4.14",
"@types/passport-jwt": "^3.0.13",
"@types/supertest": "^2.0.16",
"@types/ua-parser-js": "^0.7.39",