📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(improvement) Migrate from postmark to AWS SES (#229)
Andrii Romasiunfb21b7c47062
diff --git a/.env.example b/.env.exampleindex 36b38b30c..f454919b7 100644--- a/.env.example+++ b/.env.example@@ -33,7 +33,6 @@ API_ORIGINS=# EMAIL SERVICESMTP_HOST=localhost-SMTP_BROADCAST_HOST=localhostSMTP_PORT=1025SMTP_USER=smtp_user_nameSMTP_PASSWORD=smtp_password@@ -89,3 +88,6 @@ CLOUDFLARE_PROXY_ENABLED=false# Paypal (used for payouts to referrers)PAYPAL_CLIENT_ID=PAYPAL_CLIENT_SECRET=++# Generic+EMAIL_ACTION_ENCRYPTION_KEY=diff --git a/Dockerfile.prod b/Dockerfile.prodindex 37e9de8c8..d5ce2a423 100644--- a/Dockerfile.prod+++ b/Dockerfile.prod@@ -26,7 +26,6 @@ ENV TZ=UTC \CLICKHOUSE_DATABASE=analytics \API_ORIGINS= \SMTP_HOST=localhost \- SMTP_BROADCAST_HOST=localhost \SMTP_PORT=1025 \SMTP_USER=smtp_user_name \SMTP_PASSWORD=smtp_password \diff --git a/apps/production/src/app.module.ts b/apps/production/src/app.module.tsindex a688b9b44..a8572d89b 100644--- a/apps/production/src/app.module.ts+++ b/apps/production/src/app.module.ts@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'import { ConfigModule } from '@nestjs/config'import { ScheduleModule } from '@nestjs/schedule'import { NestjsFormDataModule } from 'nestjs-form-data'+import { MailerModule as NodeMailerModule } from '@nestjs-modules/mailer'import { I18nModule } from 'nestjs-i18n'import { UserModule } from './user/user.module'@@ -30,6 +31,22 @@ const modules = [expandVariables: true,isGlobal: true,}),+ NodeMailerModule.forRoot({+ transport: {+ sendingRate: 14,+ // pool: true, // if true - set up pooled connections against a SMTP server+ host: process.env.SMTP_HOST,+ port: Number(process.env.SMTP_PORT),+ secure: true, // if false - upgrade later with STARTTLS+ auth: {+ user: process.env.SMTP_USER,+ pass: process.env.SMTP_PASSWORD,+ },+ },+ defaults: {+ from: `"Swetrix" <${process.env.FROM_EMAIL}>`, // outgoing email ID+ },+ }),TypeOrmModule.forRoot({type: 'mysql',host: process.env.MYSQL_HOST,diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.tsindex 306826076..2487caaff 100644--- a/apps/production/src/common/constants.ts+++ b/apps/production/src/common/constants.ts@@ -7,8 +7,12 @@ import * as _toNumber from 'lodash/toNumber'// eslint-disable-next-line @typescript-eslint/no-var-requiresrequire('dotenv').config()-const { CLICKHOUSE_DATABASE, PAYPAL_CLIENT_ID, PAYPAL_CLIENT_SECRET } =- process.env+const {+ CLICKHOUSE_DATABASE,+ PAYPAL_CLIENT_ID,+ PAYPAL_CLIENT_SECRET,+ EMAIL_ACTION_ENCRYPTION_KEY,+} = process.envconst redis = new Redis(_toNumber(process.env.REDIS_PORT),@@ -211,6 +215,7 @@ export {REDIS_LOG_PERF_CACHE_KEY,CAPTCHA_SALT,CAPTCHA_ENCRYPTION_KEY,+ EMAIL_ACTION_ENCRYPTION_KEY,isDevelopment,getRedisCaptchaKey,CAPTCHA_COOKIE_KEY,diff --git a/apps/production/src/common/templates/en/customer_sign-up.html b/apps/production/src/common/templates/en/customer_sign-up.htmlindex 432896a2f..23054afa4 100644--- a/apps/production/src/common/templates/en/customer_sign-up.html+++ b/apps/production/src/common/templates/en/customer_sign-up.html@@ -1,7 +1,6 @@Hello and welcome to Swetrix! We're excited to have you with us.<br/>Swetrix is an ethical and powerful web analytics platform that respects your privacy and data.<br/><br/>-To get started, please verify your email address by clicking the link below:<br/>-<a href="{{url}}">Verify registration</a><br/><br/>+To get started, please <a href="{{url}}">click here</a> verify your email address.<br/><br/>If you’re having trouble with the link above, copy and paste the URL below directly into your web browser:<br/>{{url}}<br/><br/>If you did not request this email, feel free to ignore it :)diff --git a/apps/production/src/common/templates/en/project-report.html b/apps/production/src/common/templates/en/project-report.htmlindex 2aa3c2915..13e7a4f83 100644--- a/apps/production/src/common/templates/en/project-report.html+++ b/apps/production/src/common/templates/en/project-report.html@@ -97,8 +97,8 @@<br /><br />- You can manage email reports preferences in your <a href="https://swetrix.com/settings"- target="_blank">account settings</a>.+ Do not want to receive these emails? <a href="{{data.unsubscribeUrl}}"+ target="_blank">Click here</a> to unsubscribe.</span></font><div style="height: 30px; line-height: 30px; font-size: 28px;"> </div>diff --git a/apps/production/src/mailer/mailer.service.ts b/apps/production/src/mailer/mailer.service.tsindex 63f655e70..722d52e73 100644--- a/apps/production/src/mailer/mailer.service.ts+++ b/apps/production/src/mailer/mailer.service.ts@@ -1,7 +1,7 @@import fs = require('fs')import path = require('path')+import { MailerService as NodeMailerService } from '@nestjs-modules/mailer'import { Injectable } from '@nestjs/common'-import * as postmark from 'postmark'import handlebars from 'handlebars'import { LetterTemplate } from './letter'import { AppLoggerService } from '../logger/logger.service'@@ -104,17 +104,19 @@ const metaInfoJson = {},[LetterTemplate.UsageOverLimit]: {subject: {- en: () => 'You have exceeded your Swetrix subscription plan',+ en: () =>+ '[ACTION REQUIRED] You have exceeded your Swetrix subscription plan',},},[LetterTemplate.DashboardLockedExceedingLimits]: {subject: {- en: () => 'Your Swetrix dashboard has been locked',+ en: () => '[ACTION REQUIRED] Your Swetrix dashboard has been locked',},},[LetterTemplate.DashboardLockedPaymentFailure]: {subject: {- en: () => 'Your Swetrix dashboard has been locked due to a payment issue',+ en: () =>+ '[ACTION REQUIRED] Your Swetrix dashboard has been locked due to a payment issue',},},}@@ -143,31 +145,30 @@ handlebars.registerHelper('greater', function greater(v1, v2, options) {return options.inverse(this)})-const mailClient = new postmark.ServerClient(process.env.SMTP_PASSWORD)-@Injectable()export class MailerService {- constructor(private readonly logger: AppLoggerService) {}+ constructor(+ private readonly logger: AppLoggerService,+ private readonly nodeMailerService: NodeMailerService,+ ) {}async sendEmail(email: string,templateName: LetterTemplate,params: Params = null,- messageStream: 'broadcast' | 'outbound' = 'outbound',): Promise<void> {try {const templatePath = `${TEMPLATES_PATH}/en/${templateName}.html`const letter = fs.readFileSync(templatePath, { encoding: 'utf-8' })const subject = metaInfoJson[templateName].subject.en(params)const template = handlebars.compile(letter)- const htmlToSend = template(params)+ const html = template(params)- const message: postmark.Models.Message = {- From: process.env.FROM_EMAIL,- To: email,- Subject: subject,- HtmlBody: htmlToSend,- MessageStream: messageStream,+ const message = {+ from: process.env.FROM_EMAIL,+ to: email,+ subject,+ html,}if (process.env.SMTP_MOCK) {@@ -180,10 +181,10 @@ export class MailerService {true,)} else {- await mailClient.sendEmail(message)+ await this.nodeMailerService.sendMail(message)}- } catch (error) {- console.error(`[ERROR][MAILER] ${error}`)+ } catch (reason) {+ this.logger.error(reason, 'sendEmail', true)}}}diff --git a/apps/production/src/main.ts b/apps/production/src/main.tsindex b1f28e89a..2f4d1519c 100644--- a/apps/production/src/main.ts+++ b/apps/production/src/main.ts@@ -76,6 +76,9 @@ async function bootstrap() {app.use('/webhook', bodyParser.raw({ type: 'application/json' }))+ app.use('/webhook/sns', bodyParser.raw())+ app.use('/webhook/sns', bodyParser.text())+if (process.env.NODE_ENV === 'production') {const bot = app.get(getBotToken())app.use(bot.webhookCallback(process.env.TELEGRAM_WEBHOOK_PATH))diff --git a/apps/production/src/project/project.controller.ts b/apps/production/src/project/project.controller.tsindex fe8eb66b5..8189f289c 100644--- a/apps/production/src/project/project.controller.ts+++ b/apps/production/src/project/project.controller.ts@@ -1602,6 +1602,32 @@ export class ProjectController {return _omit(project, ['passwordHash'])}+ // Used to unsubscribe from email reports for 3rd party users (i.e. project-subscriber.entity.ts)+ @Get('/unsubscribe/:token')+ @Public()+ @ApiResponse({ status: 204 })+ async unsubscribeFromEmailReports(+ @Param('token') token: string,+ ): Promise<void> {+ this.logger.log({ token }, 'GET /project/unsubscribe/:token')++ let subscriberId++ try {+ subscriberId = this.projectService.decryptUnsubscribeKey(token)+ } catch {+ throw new NotFoundException('Unsubscribe token is invalid')+ }++ const subscriber = await this.projectService.getSubscriberById(subscriberId)++ if (!subscriber) {+ throw new NotFoundException('Unsubscribe token is invalid')+ }++ await this.projectService.removeSubscriberById(subscriberId)+ }+@Get('/ogimage/:id')@HttpCode(200)@Header('Content-Type', 'image/jpeg')diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.tsindex 96c43100c..5a8823a62 100644--- a/apps/production/src/project/project.service.ts+++ b/apps/production/src/project/project.service.ts@@ -12,6 +12,7 @@ import { Repository } from 'typeorm'import { customAlphabet } from 'nanoid'import handlebars from 'handlebars'import puppeteer from 'puppeteer'+import * as CryptoJS from 'crypto-js'import * as _isEmpty from 'lodash/isEmpty'import * as _size from 'lodash/size'import * as _join from 'lodash/join'@@ -51,6 +52,7 @@ import {getRedisUserUsageInfoKey,redisUserUsageinfoCacheTimeout,TRAFFIC_COLUMNS,+ EMAIL_ACTION_ENCRYPTION_KEY,} from '../common/constants'import { IUsageInfoRedis } from '../user/interfaces'import { ProjectSubscriber, Funnel } from './entity'@@ -922,12 +924,22 @@ export class ProjectService {})}+ async findOneSubscriber(where: Record<string, unknown>) {+ return this.projectSubscriberRepository.findOne({ where })+ }+async getSubscriber(projectId: string, subscriberId: string) {return this.projectSubscriberRepository.findOne({where: { id: subscriberId, projectId },})}+ async getSubscriberById(id: string) {+ return this.projectSubscriberRepository.findOne({+ where: { id },+ })+ }+async confirmSubscriber(projectId: string,subscriberId: string,@@ -997,6 +1009,12 @@ export class ProjectService {})}+ async removeSubscriberById(id: string) {+ await this.projectSubscriberRepository.delete({+ id,+ })+ }+async getSubscribersForReports(reportFrequency: ReportFrequency) {return this.projectSubscriberRepository.find({relations: ['project'],@@ -1018,6 +1036,23 @@ export class ProjectService {})}+ createUnsubscribeKey(subscriberId: string): string {+ return encodeURIComponent(+ CryptoJS.Rabbit.encrypt(+ subscriberId,+ EMAIL_ACTION_ENCRYPTION_KEY,+ ).toString(),+ )+ }++ decryptUnsubscribeKey(token: string): string {+ const bytes = CryptoJS.Rabbit.decrypt(+ decodeURIComponent(token),+ EMAIL_ACTION_ENCRYPTION_KEY,+ )+ return bytes.toString(CryptoJS.enc.Utf8)+ }+async transferProject(projectId: string,name: string,diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.tsindex 209d6ab69..357537efa 100644--- a/apps/production/src/task-manager/task-manager.service.ts+++ b/apps/production/src/task-manager/task-manager.service.ts@@ -1,6 +1,7 @@import { Injectable } from '@nestjs/common'import { Cron, CronExpression } from '@nestjs/schedule'import { IsNull, LessThan, In, Not, Between, MoreThan, Like } from 'typeorm'+import { ConfigService } from '@nestjs/config'import * as Paypal from '@paypal/payouts-sdk'import * as bcrypt from 'bcrypt'import * as dayjs from 'dayjs'@@ -231,6 +232,24 @@ const getUsersThatExceedContinuously = (return exceedingUsers as User & { usage: any[] }[]}+const EMAIL_REPORTS_MAP = {+ [ReportFrequency.WEEKLY]: {+ type: 'w',+ dayjsParams: [1, 'w'],+ analyticsParam: '7d',+ },+ [ReportFrequency.MONTHLY]: {+ type: 'M',+ dayjsParams: [1, 'M'],+ analyticsParam: '4w',+ },+ [ReportFrequency.QUARTERLY]: {+ type: 'M',+ dayjsParams: [3, 'M'],+ analyticsParam: '3M',+ },+}+@Injectable()export class TaskManagerService {constructor(@@ -244,8 +263,152 @@ export class TaskManagerService {private readonly logger: AppLoggerService,private readonly telegramService: TelegramService,private readonly payoutsService: PayoutsService,+ private readonly configService: ConfigService,) {}+ generateUnsubscribeUrl(+ id: string,+ type: '3rdparty' | 'user-reports',+ ): string {+ if (type === '3rdparty') {+ const token = this.projectService.createUnsubscribeKey(id)+ return `${this.configService.get(+ 'CLIENT_URL',+ )}/3rd-party-unsubscribe/${token}`+ }++ const token = this.userService.createUnsubscribeKey(id)+ return `${this.configService.get(+ 'CLIENT_URL',+ )}/reports-unsubscribe/${token}`+ }++ async handleUserReports(+ reportFrequency:+ | ReportFrequency.WEEKLY+ | ReportFrequency.MONTHLY+ | ReportFrequency.QUARTERLY,+ ): Promise<void> {+ const params = EMAIL_REPORTS_MAP[reportFrequency]++ const users = await this.userService.find({+ where: {+ reportFrequency,+ planCode: Not(PlanCode.none),+ dashboardBlockReason: IsNull(),+ },+ relations: ['projects'],+ select: ['id', 'email'],+ })+ const now = dayjs.utc().format('DD.MM.YYYY')+ const timeAgo = dayjs+ .utc()+ // @ts-ignore+ .subtract(...params.dayjsParams)+ .format('DD.MM.YYYY')+ const date = `${timeAgo} - ${now}`+ const tip = getRandomTip()++ const promises = _map(users, async user => {+ const { id, email, projects } = user++ if (_isEmpty(projects) || _isNull(projects)) {+ return+ }++ const unsubscribeUrl = this.generateUnsubscribeUrl(id, 'user-reports')++ const ids = _map(projects, p => p.id)+ const data = this.analyticsService.convertSummaryToObsoleteFormat(+ await this.analyticsService.getAnalyticsSummary(+ ids,+ params.analyticsParam,+ ),+ )++ const result = {+ type: params.type,+ date,+ projects: _map(ids, (pid, index) => ({+ data: data[pid],+ name: projects[index].name,+ })),+ tip,+ unsubscribeUrl,+ }++ await this.mailerService.sendEmail(+ email,+ LetterTemplate.ProjectReport,+ result,+ )+ })++ await Promise.allSettled(promises).catch(reason => {+ this.logger.error(+ `[CRON WORKER](handleUserReports) Frequency: ${reportFrequency}; Error occured: ${reason}`,+ )+ })+ }++ async handleSubscriberReports(+ reportFrequency:+ | ReportFrequency.WEEKLY+ | ReportFrequency.MONTHLY+ | ReportFrequency.QUARTERLY,+ ): Promise<void> {+ const params = EMAIL_REPORTS_MAP[reportFrequency]++ const subscribers =+ await this.projectService.getSubscribersForReports(reportFrequency)+ const now = dayjs.utc().format('DD.MM.YYYY')+ const timeAgo = dayjs+ .utc()+ // @ts-ignore+ .subtract(...params.dayjsParams)+ .format('DD.MM.YYYY')+ const date = `${timeAgo} - ${now}`+ const tip = getRandomTip()++ const promises = _map(subscribers, async subscriber => {+ const { id, email } = subscriber+ const projects = await this.projectService.getSubscriberProjects(id)++ const unsubscribeUrl = this.generateUnsubscribeUrl(id, '3rdparty')++ const ids = projects.map(project => project.id)+ const data = this.analyticsService.convertSummaryToObsoleteFormat(+ await this.analyticsService.getAnalyticsSummary(+ ids,+ params.analyticsParam,+ ),+ )++ const result = {+ type: params.type,+ date,+ projects: _map(ids, (pid, index) => ({+ data: data[pid],+ name: projects[index].name,+ })),+ tip,+ unsubscribeUrl,+ }++ await this.mailerService.sendEmail(+ email,+ LetterTemplate.ProjectReport,+ result,+ )+ })++ await Promise.allSettled(promises).catch(reason => {+ this.logger.error(+ `[CRON WORKER](handleSubscriberReports) Frequency: ${reportFrequency}; Error: ${reason}`,+ )+ })+ }+@Cron(CronExpression.EVERY_MINUTE)async saveLogData(): Promise<void> {const data = await redis.lrange(REDIS_LOG_DATA_CACHE_KEY, 0, -1)@@ -547,7 +710,6 @@ export class TaskManagerService {email,LetterTemplate.TierWarning,emailParams,- 'broadcast',)await this.userService.update(id, {evWarningSentOn: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),@@ -595,298 +757,37 @@ export class TaskManagerService {// EVERY SUNDAY AT 2:30 AM@Cron('30 02 * * 0')async weeklyReportsHandler(): Promise<void> {- const users = await this.userService.find({- where: {- reportFrequency: ReportFrequency.WEEKLY,- planCode: Not(PlanCode.none),- dashboardBlockReason: IsNull(),- },- relations: ['projects'],- select: ['email'],- })- const now = dayjs.utc().format('DD.MM.YYYY')- const weekAgo = dayjs.utc().subtract(1, 'w').format('DD.MM.YYYY')- const date = `${weekAgo} - ${now}`- const tip = getRandomTip()-- const promises = _map(users, async user => {- const { email, projects } = user-- if (_isEmpty(projects) || _isNull(projects)) {- return- }-- const ids = _map(projects, p => p.id)- const data = this.analyticsService.convertSummaryToObsoleteFormat(- await this.analyticsService.getAnalyticsSummary(ids, '7d'),- )-- const result = {- type: 'w', // week- date,- projects: _map(ids, (pid, index) => ({- data: data[pid],- name: projects[index].name,- })),- tip,- }-- await this.mailerService.sendEmail(- email,- LetterTemplate.ProjectReport,- result,- 'broadcast',- )- })-- await Promise.allSettled(promises).catch(reason => {- this.logger.error(- `[CRON WORKER](weeklyReportsHandler) Error occured: ${reason}`,- )- })+ await this.handleUserReports(ReportFrequency.WEEKLY)}// ON THE FIRST DAY OF EVERY MONTH AT 2 AM@Cron('0 02 1 * *')async monthlyReportsHandler(): Promise<void> {- const users = await this.userService.find({- where: {- reportFrequency: ReportFrequency.MONTHLY,- planCode: Not(PlanCode.none),- dashboardBlockReason: IsNull(),- },- relations: ['projects'],- select: ['email'],- })- const now = dayjs.utc().format('DD.MM.YYYY')- const weekAgo = dayjs.utc().subtract(1, 'M').format('DD.MM.YYYY')- const date = `${weekAgo} - ${now}`- const tip = getRandomTip()-- const promises = _map(users, async user => {- const { email, projects } = user-- if (_isEmpty(projects) || _isNull(projects)) {- return- }-- const ids = _map(projects, p => p.id)- const data = this.analyticsService.convertSummaryToObsoleteFormat(- await this.analyticsService.getAnalyticsSummary(ids, '4w'),- )-- const result = {- type: 'M', // month- date,- projects: _map(ids, (pid, index) => ({- data: data[pid],- name: projects[index].name,- })),- tip,- }-- await this.mailerService.sendEmail(- email,- LetterTemplate.ProjectReport,- result,- 'broadcast',- )- })-- await Promise.allSettled(promises).catch(reason => {- this.logger.error(- `[CRON WORKER](monthlyReportsHandler) Error occured: ${reason}`,- )- })+ await this.handleUserReports(ReportFrequency.MONTHLY)}@Cron(CronExpression.EVERY_QUARTER)async quarterlyReportsHandler(): Promise<void> {- const users = await this.userService.find({- where: {- reportFrequency: ReportFrequency.QUARTERLY,- planCode: Not(PlanCode.none),- dashboardBlockReason: IsNull(),- },- relations: ['projects'],- select: ['email'],- })- const now = dayjs.utc().format('DD.MM.YYYY')- const quarterAgo = dayjs.utc().subtract(3, 'M').format('DD.MM.YYYY')- const date = `${quarterAgo} - ${now}`- const tip = getRandomTip()-- const promises = _map(users, async user => {- const { email, projects } = user-- if (_isEmpty(projects) || _isNull(projects)) {- return- }-- const ids = _map(projects, p => p.id)- const data = this.analyticsService.convertSummaryToObsoleteFormat(- await this.analyticsService.getAnalyticsSummary(ids, '3M'),- )-- const result = {- type: 'Q', // quarterly- date,- projects: _map(ids, (pid, index) => ({- data: data[pid],- name: projects[index].name,- })),- tip,- }-- await this.mailerService.sendEmail(- email,- LetterTemplate.ProjectReport,- result,- 'broadcast',- )- })-- await Promise.allSettled(promises).catch(reason => {- this.logger.error(- `[CRON WORKER](quarterlyReportsHandler) Error occured: ${reason}`,- )- })+ await this.handleUserReports(ReportFrequency.QUARTERLY)}// EMAIL REPORTS, BUT FOR MULTIPLE PROJECT SUBSCRIBERS@Cron(CronExpression.EVERY_QUARTER)async handleQuarterlyReports(): Promise<void> {- const subscribers = await this.projectService.getSubscribersForReports(- ReportFrequency.QUARTERLY,- )- const now = dayjs.utc().format('DD.MM.YYYY')- const quarterAgo = dayjs.utc().subtract(3, 'M').format('DD.MM.YYYY')- const date = `${quarterAgo} - ${now}`- const tip = getRandomTip()-- const promises = _map(subscribers, async subscriber => {- const { id, email } = subscriber- const projects = await this.projectService.getSubscriberProjects(id)-- const ids = projects.map(project => project.id)- const data = this.analyticsService.convertSummaryToObsoleteFormat(- await this.analyticsService.getAnalyticsSummary(ids, '3M'),- )-- const result = {- type: 'Q', // quarter- date,- projects: _map(ids, (pid, index) => ({- data: data[pid],- name: projects[index].name,- })),- tip,- }-- await this.mailerService.sendEmail(- email,- LetterTemplate.ProjectReport,- result,- 'broadcast',- )- })-- await Promise.allSettled(promises).catch(reason => {- this.logger.error(- `[CRON WORKER](handleQuarterlyReports) Error occured: ${reason}`,- )- })+ await this.handleSubscriberReports(ReportFrequency.QUARTERLY)}// ON THE FIRST DAY OF EVERY MONTH AT 3 AM@Cron('0 03 1 * *')async handleMonthlyReports(): Promise<void> {- const subscribers = await this.projectService.getSubscribersForReports(- ReportFrequency.MONTHLY,- )- const now = dayjs.utc().format('DD.MM.YYYY')- const weekAgo = dayjs.utc().subtract(1, 'M').format('DD.MM.YYYY')- const date = `${weekAgo} - ${now}`- const tip = getRandomTip()-- const promises = _map(subscribers, async subscriber => {- const { id, email } = subscriber- const projects = await this.projectService.getSubscriberProjects(id)-- const ids = projects.map(project => project.id)- const data = this.analyticsService.convertSummaryToObsoleteFormat(- await this.analyticsService.getAnalyticsSummary(ids, '4w'),- )-- const result = {- type: 'M', // month- date,- projects: _map(ids, (pid, index) => ({- data: data[pid],- name: projects[index].name,- })),- tip,- }-- await this.mailerService.sendEmail(- email,- LetterTemplate.ProjectReport,- result,- 'broadcast',- )- })-- await Promise.allSettled(promises).catch(reason => {- this.logger.error(- `[CRON WORKER](handleMonthlyReports) Error occured: ${reason}`,- )- })+ await this.handleSubscriberReports(ReportFrequency.MONTHLY)}// EVERY SUNDAY AT 3 AM@Cron('0 03 * * 0')async handleWeeklyReports(): Promise<void> {- const subscribers = await this.projectService.getSubscribersForReports(- ReportFrequency.WEEKLY,- )- const now = dayjs.utc().format('DD.MM.YYYY')- const weekAgo = dayjs.utc().subtract(1, 'w').format('DD.MM.YYYY')- const date = `${weekAgo} - ${now}`- const tip = getRandomTip()-- const promises = _map(subscribers, async subscriber => {- const { id, email } = subscriber- const projects = await this.projectService.getSubscriberProjects(id)-- const ids = projects.map(project => project.id)- const data = this.analyticsService.convertSummaryToObsoleteFormat(- await this.analyticsService.getAnalyticsSummary(ids, '7d'),- )-- const result = {- type: 'w', // week- date,- projects: _map(ids, (pid, index) => ({- data: data[pid],- name: projects[index].name,- })),- tip,- }-- await this.mailerService.sendEmail(- email,- LetterTemplate.ProjectReport,- result,- 'broadcast',- )- })-- await Promise.allSettled(promises).catch(reason => {- this.logger.error(- `[CRON WORKER](handleWeeklyReports) Error occured: ${reason}`,- )- })+ await this.handleSubscriberReports(ReportFrequency.WEEKLY)}@Cron(CronExpression.EVERY_10_MINUTES)diff --git a/apps/production/src/user/user.controller.ts b/apps/production/src/user/user.controller.tsindex 53a157a78..fa9f98d71 100644--- a/apps/production/src/user/user.controller.ts+++ b/apps/production/src/user/user.controller.ts@@ -16,6 +16,7 @@ import {Headers,Ip,Patch,+ NotFoundException,} from '@nestjs/common'import { Request } from 'express'import { ApiTags, ApiQuery, ApiResponse, ApiBearerAuth } from '@nestjs/swagger'@@ -72,6 +73,7 @@ import {generateRefCode,} from '../common/utils'import { IUsageInfo, IMetaInfo } from './interfaces'+import { ReportFrequency } from '../project/enums'dayjs.extend(utc)@@ -899,4 +901,34 @@ export class UserController {return this.userService.previewSubscription(id, planId)}++ // Used to unsubscribe from email reports+ @Get('/unsubscribe/:token')+ @Public()+ @ApiResponse({ status: 204 })+ async unsubscribeFromEmailReports(+ @Param('token') token: string,+ ): Promise<void> {+ this.logger.log({ token }, 'GET /project/unsubscribe/:token')++ let userId++ try {+ userId = this.userService.decryptUnsubscribeKey(token)+ } catch {+ throw new NotFoundException('Unsubscribe token is invalid')+ }++ const user = await this.userService.findOneWhere({+ id: userId,+ })++ if (!user) {+ throw new NotFoundException('Unsubscribe token is invalid')+ }++ await this.userService.update(userId, {+ reportFrequency: ReportFrequency.NEVER,+ })+ }}diff --git a/apps/production/src/user/user.service.ts b/apps/production/src/user/user.service.tsindex 2c65c5700..ea1f92a3c 100644--- a/apps/production/src/user/user.service.ts+++ b/apps/production/src/user/user.service.ts@@ -6,6 +6,7 @@ import {import { InjectRepository } from '@nestjs/typeorm'import { In, Repository } from 'typeorm'import axios from 'axios'+import * as CryptoJS from 'crypto-js'import * as dayjs from 'dayjs'import * as _isEmpty from 'lodash/isEmpty'import * as _size from 'lodash/size'@@ -28,6 +29,7 @@ import { RefreshToken } from './entities/refresh-token.entity'import { DeleteFeedback } from './entities/delete-feedback.entity'import { UserGoogleDTO } from './dto/user-google.dto'import { UserGithubDTO } from './dto/user-github.dto'+import { EMAIL_ACTION_ENCRYPTION_KEY } from '../common/constants'const EUR = {symbol: '€',@@ -508,6 +510,20 @@ export class UserService {)}+ createUnsubscribeKey(userId: string): string {+ return encodeURIComponent(+ CryptoJS.Rabbit.encrypt(userId, EMAIL_ACTION_ENCRYPTION_KEY).toString(),+ )+ }++ decryptUnsubscribeKey(token: string): string {+ const bytes = CryptoJS.Rabbit.decrypt(+ decodeURIComponent(token),+ EMAIL_ACTION_ENCRYPTION_KEY,+ )+ return bytes.toString(CryptoJS.enc.Utf8)+ }+async getReferralsList(user: User): Promise<Partial<User>[]> {return this.usersRepository.createQueryBuilder('user')diff --git a/apps/production/src/webhook/webhook.controller.ts b/apps/production/src/webhook/webhook.controller.tsindex bf8b408f0..2c82c22ae 100644--- a/apps/production/src/webhook/webhook.controller.ts+++ b/apps/production/src/webhook/webhook.controller.ts@@ -39,6 +39,44 @@ export class WebhookController {private readonly mailerService: MailerService,) {}+ // AWS SNS webhook+ @Post('/sns')+ @HttpCode(200)+ async snsWebhook(@Body() body): Promise<any> {+ const json = typeof body === 'string' ? JSON.parse(body) : body++ await this.webhookService.verifySNSRequest(json)++ const { Type, Message } = json++ if (Type !== 'Notification') {+ return+ }++ const message = typeof Message === 'string' ? JSON.parse(Message) : Message++ switch (message.eventType) {+ case 'Bounce': {+ const { emailAddress } = message.bounce.bouncedRecipients[0]++ await this.webhookService.unsubscribeByEmail(emailAddress)++ break+ }++ case 'Complaint': {+ const { emailAddress } = message.complaint.complainedRecipients[0]++ await this.webhookService.unsubscribeByEmail(emailAddress)++ break+ }++ default:+ }+ }++ // Paddle - payment processor webhook@Post('/paddle')@HttpCode(200)async paddleWebhook(diff --git a/apps/production/src/webhook/webhook.service.ts b/apps/production/src/webhook/webhook.service.tsindex 747b4c226..b4dcfd697 100644--- a/apps/production/src/webhook/webhook.service.ts+++ b/apps/production/src/webhook/webhook.service.ts@@ -1,5 +1,6 @@/* eslint-disable no-param-reassign, no-prototype-builtins */import * as crypto from 'crypto'+import * as Validator from 'sns-payload-validator'import {Injectable,BadRequestException,@@ -11,8 +12,11 @@ import * as _keys from 'lodash/keys'import * as _isArray from 'lodash/isArray'import { AppLoggerService } from '../logger/logger.service'import { PayoutsService } from '../payouts/payouts.service'+import { ProjectService } from '../project/project.service'+import { UserService } from '../user/user.service'import { PayoutStatus } from '../payouts/entities/payouts.entity'import { User } from '../user/entities/user.entity'+import { ReportFrequency } from '../project/enums'const PADDLE_PUB_KEY = `-----BEGIN PUBLIC KEY-----MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAqFcHslKkXcJlTYg4FL6j@@ -40,11 +44,15 @@ const paddleWhitelistIPs = ['3.208.120.145',]+const validator = new Validator()+@Injectable()export class WebhookService {constructor(private readonly logger: AppLoggerService,private readonly payoutsService: PayoutsService,+ private readonly userService: UserService,+ private readonly projectService: ProjectService,) {}public ksort(obj: Record<string, unknown>): Record<string, unknown> {@@ -125,4 +133,43 @@ export class WebhookService {},)}++ async verifySNSRequest(payload: any): Promise<void> {+ try {+ await validator.validate(payload)+ } catch {+ throw new BadRequestException('Webhook signature verification failed')+ }+ }++ async unsubscribeByEmail(email: string): Promise<void> {+ const user = await this.userService.findOneWhere({+ email,+ })++ // checking if the bounce originates from admin email reports+ if (user) {+ await this.userService.update(user.id, {+ reportFrequency: ReportFrequency.NEVER,+ })+ return+ }++ // checking if the bounce originates from project subscriber email reports+ const subscribtion = await this.projectService.findOneSubscriber({+ email,+ })++ if (subscribtion) {+ await this.projectService.removeSubscriberById(subscribtion.id)+ }++ // if it's some other kind of bounce, we can ignore it because it originates from other transactional emails+ // but I'll log it here for now+ this.logger.log(+ `Received an email notification, but it originates from other transactional emails. Email: ${email}`,+ 'POST /webhook/ses',+ true,+ )+ }}diff --git a/docker-compose-prod.yml b/docker-compose-prod.ymlindex 64877bc7b..ee25b6d28 100644--- a/docker-compose-prod.yml+++ b/docker-compose-prod.yml@@ -29,7 +29,6 @@ services:- CLICKHOUSE_HOST=http://clickhouse- API_ORIGINS=- SMTP_HOST=localhost- - SMTP_BROADCAST_HOST=localhost- SMTP_PORT=1025- SMTP_USER=smtp_user_name- SMTP_PASSWORD=smtp_passworddiff --git a/package.json b/package.jsonindex 8fa8e6951..b7411ffdd 100644--- a/package.json+++ b/package.json@@ -27,6 +27,7 @@},"dependencies": {"@faker-js/faker": "^8.3.1",+ "@nestjs-modules/mailer": "^1.11.2","@nestjs/axios": "^3.0.1","@nestjs/common": "^10.2.10","@nestjs/config": "^3.1.1",@@ -71,17 +72,18 @@"nestjs-form-data": "^1.9.4","nestjs-i18n": "^10.4.0","nestjs-telegraf": "^2.7.0",+ "nodemailer": "^6.9.13","otplib": "^12.0.1","passport": "^0.7.0","passport-headerapikey": "^1.2.2","passport-jwt": "^4.0.1","php-serialize": "^4.1.1",- "postmark": "^4.0.2","puppeteer": "^21.6.0","randomstring": "^1.3.0","reflect-metadata": "^0.1.14","rimraf": "^3.0.2","rxjs": "^7.8.1",+ "sns-payload-validator": "^2.1.0","svg-captcha": "^1.4.0","telegraf": "^4.15.3","typeorm": "^0.2.45",@@ -98,6 +100,7 @@"@types/jest": "^29.5.11","@types/multer": "^1.4.11","@types/node": "^20.10.4",+ "@types/nodemailer": "^6.4.14","@types/passport-jwt": "^3.0.13","@types/supertest": "^2.0.16","@types/ua-parser-js": "^0.7.39",