Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/add google sheets as data source (#620)

Thomas Kaul075431d868e9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 395fdc199c..7f3c5eda3c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,10 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
+### Added
+
+- Added `GOOGLE_SHEETS` as a new data source type
+
### Changed
- Excluded the url pattern of shared portfolios in the `robots.txt` file
+### Todo
+
+- Apply data migration (`yarn database:migrate`)
+
## 1.100.0 - 05.01.2022
### Added
diff --git a/apps/api/src/app/symbol/symbol.controller.ts b/apps/api/src/app/symbol/symbol.controller.ts
index a5c31fe6a6..bbe582ceaa 100644
--- a/apps/api/src/app/symbol/symbol.controller.ts
+++ b/apps/api/src/app/symbol/symbol.controller.ts
@@ -13,7 +13,7 @@ import {
} from '@nestjs/common';
import { REQUEST } from '@nestjs/core';
import { AuthGuard } from '@nestjs/passport';
-import { DataSource, MarketData } from '@prisma/client';
+import { DataSource } from '@prisma/client';
import { StatusCodes, getReasonPhrase } from 'http-status-codes';
import { isDate, isEmpty } from 'lodash';
@@ -37,8 +37,7 @@ export class SymbolController {
@Query() { query = '' }
): Promise<{ items: LookupItem[] }> {
try {
- const encodedQuery = encodeURIComponent(query.toLowerCase());
- return this.symbolService.lookup(encodedQuery);
+ return this.symbolService.lookup(query.toLowerCase());
} catch {
throw new HttpException(
getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR),
diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts
index b120d7ea21..8c95ce9474 100644
--- a/apps/api/src/app/symbol/symbol.service.ts
+++ b/apps/api/src/app/symbol/symbol.service.ts
@@ -93,32 +93,6 @@ export class SymbolService {
try {
const { items } = await this.dataProviderService.search(aQuery);
results.items = items;
-
- // Add custom symbols
- const ghostfolioSymbolProfiles =
- await this.prismaService.symbolProfile.findMany({
- select: {
- currency: true,
- dataSource: true,
- name: true,
- symbol: true
- },
- where: {
- AND: [
- {
- dataSource: DataSource.GHOSTFOLIO,
- name: {
- startsWith: aQuery
- }
- }
- ]
- }
- });
-
- for (const ghostfolioSymbolProfile of ghostfolioSymbolProfiles) {
- results.items.push(ghostfolioSymbolProfile);
- }
-
return results;
} catch (error) {
Logger.error(error);
diff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.ts
index b2d9c65fba..0c358f0991 100644
--- a/apps/api/src/services/configuration.service.ts
+++ b/apps/api/src/services/configuration.service.ts
@@ -13,6 +13,7 @@ export class ConfigurationService {
ACCESS_TOKEN_SALT: str(),
ALPHA_VANTAGE_API_KEY: str({ default: '' }),
CACHE_TTL: num({ default: 1 }),
+ DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),
DATA_SOURCES: json({ default: JSON.stringify([DataSource.YAHOO]) }),
ENABLE_FEATURE_BLOG: bool({ default: false }),
ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),
@@ -25,6 +26,9 @@ export class ConfigurationService {
ENABLE_FEATURE_SYSTEM_MESSAGE: bool({ default: false }),
GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }),
GOOGLE_SECRET: str({ default: 'dummySecret' }),
+ GOOGLE_SHEETS_ACCOUNT: str({ default: '' }),
+ GOOGLE_SHEETS_ID: str({ default: '' }),
+ GOOGLE_SHEETS_PRIVATE_KEY: str({ default: '' }),
JWT_SECRET_KEY: str({}),
MAX_ITEM_IN_CACHE: num({ default: 9999 }),
MAX_ORDERS_TO_IMPORT: num({ default: Number.MAX_SAFE_INTEGER }),
diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts
index 625bbc6278..838f1ae6ec 100644
--- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts
+++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts
@@ -88,13 +88,13 @@ export class AlphaVantageService implements DataProviderInterface {
return DataSource.ALPHA_VANTAGE;
}
- public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
- const result = await this.alphaVantage.data.search(aSymbol);
+ public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
+ const result = await this.alphaVantage.data.search(aQuery);
return {
items: result?.bestMatches?.map((bestMatch) => {
return {
- dataSource: DataSource.ALPHA_VANTAGE,
+ dataSource: this.getName(),
name: bestMatch['2. name'],
symbol: bestMatch['1. symbol']
};
diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts
index 37afb3abf5..c055709327 100644
--- a/apps/api/src/services/data-provider/data-provider.module.ts
+++ b/apps/api/src/services/data-provider/data-provider.module.ts
@@ -1,6 +1,7 @@
import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
+import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service';
import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
@@ -21,12 +22,14 @@ import { DataProviderService } from './data-provider.service';
AlphaVantageService,
DataProviderService,
GhostfolioScraperApiService,
+ GoogleSheetsService,
RakutenRapidApiService,
YahooFinanceService,
{
inject: [
AlphaVantageService,
GhostfolioScraperApiService,
+ GoogleSheetsService,
RakutenRapidApiService,
YahooFinanceService
],
@@ -34,11 +37,13 @@ import { DataProviderService } from './data-provider.service';
useFactory: (
alphaVantageService,
ghostfolioScraperApiService,
+ googleSheetsService,
rakutenRapidApiService,
yahooFinanceService
) => [
alphaVantageService,
ghostfolioScraperApiService,
+ googleSheetsService,
rakutenRapidApiService,
yahooFinanceService
]
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index 15a8a6efb0..98237af8bb 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -149,13 +149,13 @@ export class DataProviderService {
return result;
}
- public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
+ public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
const promises: Promise<{ items: LookupItem[] }>[] = [];
let lookupItems: LookupItem[] = [];
for (const dataSource of this.configurationService.get('DATA_SOURCES')) {
promises.push(
- this.getDataProvider(DataSource[dataSource]).search(aSymbol)
+ this.getDataProvider(DataSource[dataSource]).search(aQuery)
);
}
@@ -176,7 +176,7 @@ export class DataProviderService {
}
public getPrimaryDataSource(): DataSource {
- return DataSource[this.configurationService.get('DATA_SOURCES')[0]];
+ return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')];
}
private getDataProvider(providerName: DataSource) {
diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
index 77d406fdb6..a34f7cf926 100644
--- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
+++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
@@ -1,4 +1,10 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
+import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
+import {
+ IDataProviderHistoricalResponse,
+ IDataProviderResponse,
+ MarketState
+} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
import {
@@ -13,13 +19,6 @@ import * as bent from 'bent';
import * as cheerio from 'cheerio';
import { format } from 'date-fns';
-import {
- IDataProviderHistoricalResponse,
- IDataProviderResponse,
- MarketState
-} from '../../interfaces/interfaces';
-import { DataProviderInterface } from '../interfaces/data-provider.interface';
-
@Injectable()
export class GhostfolioScraperApiService implements DataProviderInterface {
private static NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g;
@@ -59,7 +58,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
[symbol]: {
marketPrice,
currency: symbolProfile?.currency,
- dataSource: DataSource.GHOSTFOLIO,
+ dataSource: this.getName(),
marketState: MarketState.delayed
}
};
@@ -116,8 +115,35 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
return DataSource.GHOSTFOLIO;
}
- public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
- return { items: [] };
+ public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
+ const items = await this.prismaService.symbolProfile.findMany({
+ select: {
+ currency: true,
+ dataSource: true,
+ name: true,
+ symbol: true
+ },
+ where: {
+ OR: [
+ {
+ dataSource: this.getName(),
+ name: {
+ mode: 'insensitive',
+ startsWith: aQuery
+ }
+ },
+ {
+ dataSource: this.getName(),
+ symbol: {
+ mode: 'insensitive',
+ startsWith: aQuery
+ }
+ }
+ ]
+ }
+ });
+
+ return { items };
}
private extractNumberFromString(aString: string): number {
diff --git a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
new file mode 100644
index 0000000000..81b369279a
--- /dev/null
+++ b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
@@ -0,0 +1,172 @@
+import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
+import {
+ IDataProviderHistoricalResponse,
+ IDataProviderResponse,
+ MarketState
+} from '@ghostfolio/api/services/interfaces/interfaces';
+import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { Granularity } from '@ghostfolio/common/types';
+import { Injectable, Logger } from '@nestjs/common';
+import { DataSource } from '@prisma/client';
+import { format } from 'date-fns';
+import { GoogleSpreadsheet } from 'google-spreadsheet';
+
+@Injectable()
+export class GoogleSheetsService implements DataProviderInterface {
+ public constructor(
+ private readonly configurationService: ConfigurationService,
+ private readonly prismaService: PrismaService,
+ private readonly symbolProfileService: SymbolProfileService
+ ) {}
+
+ public canHandle(symbol: string) {
+ return true;
+ }
+
+ public async get(
+ aSymbols: string[]
+ ): Promise<{ [symbol: string]: IDataProviderResponse }> {
+ if (aSymbols.length <= 0) {
+ return {};
+ }
+
+ try {
+ const [symbol] = aSymbols;
+ const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
+ [symbol]
+ );
+
+ const sheet = await this.getSheet({
+ sheetId: this.configurationService.get('GOOGLE_SHEETS_ID'),
+ symbol
+ });
+ const marketPrice = parseFloat(
+ (await sheet.getCellByA1('B1').value) as string
+ );
+
+ return {
+ [symbol]: {
+ marketPrice,
+ currency: symbolProfile?.currency,
+ dataSource: this.getName(),
+ marketState: MarketState.delayed
+ }
+ };
+ } catch (error) {
+ Logger.error(error);
+ }
+
+ return {};
+ }
+
+ public async getHistorical(
+ aSymbols: string[],
+ aGranularity: Granularity = 'day',
+ from: Date,
+ to: Date
+ ): Promise<{
+ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
+ }> {
+ if (aSymbols.length <= 0) {
+ return {};
+ }
+
+ try {
+ const [symbol] = aSymbols;
+
+ const sheet = await this.getSheet({
+ symbol,
+ sheetId: this.configurationService.get('GOOGLE_SHEETS_ID')
+ });
+
+ const rows = await sheet.getRows();
+
+ const historicalData: {
+ [date: string]: IDataProviderHistoricalResponse;
+ } = {};
+
+ rows
+ .filter((row, index) => {
+ return index >= 1;
+ })
+ .forEach((row) => {
+ const date = new Date(row._rawData[0]);
+ const close = parseFloat(row._rawData[1]);
+
+ historicalData[format(date, DATE_FORMAT)] = { marketPrice: close };
+ });
+
+ return {
+ [symbol]: historicalData
+ };
+ } catch (error) {
+ Logger.error(error);
+ }
+
+ return {};
+ }
+
+ public getName(): DataSource {
+ return DataSource.GOOGLE_SHEETS;
+ }
+
+ public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
+ const items = await this.prismaService.symbolProfile.findMany({
+ select: {
+ currency: true,
+ dataSource: true,
+ name: true,
+ symbol: true
+ },
+ where: {
+ OR: [
+ {
+ dataSource: this.getName(),
+ name: {
+ mode: 'insensitive',
+ startsWith: aQuery
+ }
+ },
+ {
+ dataSource: this.getName(),
+ symbol: {
+ mode: 'insensitive',
+ startsWith: aQuery
+ }
+ }
+ ]
+ }
+ });
+
+ return { items };
+ }
+
+ private async getSheet({
+ sheetId,
+ symbol
+ }: {
+ sheetId: string;
+ symbol: string;
+ }) {
+ const doc = new GoogleSpreadsheet(sheetId);
+
+ await doc.useServiceAccountAuth({
+ client_email: this.configurationService.get('GOOGLE_SHEETS_ACCOUNT'),
+ private_key: this.configurationService
+ .get('GOOGLE_SHEETS_PRIVATE_KEY')
+ .replace(/\\n/g, '\n')
+ });
+
+ await doc.loadInfo();
+
+ const sheet = doc.sheetsByTitle[symbol];
+
+ await sheet.loadCells();
+
+ return sheet;
+ }
+}
diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
index 5f99c86147..c5cf4c3309 100644
--- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
+++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
@@ -1,11 +1,10 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
-import { Granularity } from '@ghostfolio/common/types';
-import { DataSource } from '@prisma/client';
-
import {
IDataProviderHistoricalResponse,
IDataProviderResponse
-} from '../../interfaces/interfaces';
+} from '@ghostfolio/api/services/interfaces/interfaces';
+import { Granularity } from '@ghostfolio/common/types';
+import { DataSource } from '@prisma/client';
export interface DataProviderInterface {
canHandle(symbol: string): boolean;
@@ -23,5 +22,5 @@ export interface DataProviderInterface {
getName(): DataSource;
- search(aSymbol: string): Promise<{ items: LookupItem[] }>;
+ search(aQuery: string): Promise<{ items: LookupItem[] }>;
}
diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts
index dea52e74d6..bdfc147dd2 100644
--- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts
+++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts
@@ -45,7 +45,7 @@ export class RakutenRapidApiService implements DataProviderInterface {
return {
[ghostfolioFearAndGreedIndexSymbol]: {
currency: undefined,
- dataSource: DataSource.RAKUTEN,
+ dataSource: this.getName(),
marketPrice: fgi.now.value,
marketState: MarketState.open,
name: RakutenRapidApiService.FEAR_AND_GREED_INDEX_NAME
@@ -85,7 +85,7 @@ export class RakutenRapidApiService implements DataProviderInterface {
await this.prismaService.marketData.create({
data: {
symbol,
- dataSource: DataSource.RAKUTEN,
+ dataSource: this.getName(),
date: subWeeks(getToday(), 1),
marketPrice: fgi.oneWeekAgo.value
}
@@ -94,7 +94,7 @@ export class RakutenRapidApiService implements DataProviderInterface {
await this.prismaService.marketData.create({
data: {
symbol,
- dataSource: DataSource.RAKUTEN,
+ dataSource: this.getName(),
date: subMonths(getToday(), 1),
marketPrice: fgi.oneMonthAgo.value
}
@@ -103,7 +103,7 @@ export class RakutenRapidApiService implements DataProviderInterface {
await this.prismaService.marketData.create({
data: {
symbol,
- dataSource: DataSource.RAKUTEN,
+ dataSource: this.getName(),
date: subYears(getToday(), 1),
marketPrice: fgi.oneYearAgo.value
}
@@ -129,7 +129,7 @@ export class RakutenRapidApiService implements DataProviderInterface {
return DataSource.RAKUTEN;
}
- public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
+ public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
return { items: [] };
}
diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
index 0717a699e1..2b4fe8f92f 100644
--- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
+++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
@@ -103,7 +103,7 @@ export class YahooFinanceService implements DataProviderInterface {
assetClass,
assetSubClass,
currency: value.price?.currency,
- dataSource: DataSource.YAHOO,
+ dataSource: this.getName(),
exchange: this.parseExchange(value.price?.exchangeName),
marketState:
value.price?.marketState === 'REGULAR' ||
@@ -221,12 +221,14 @@ export class YahooFinanceService implements DataProviderInterface {
return DataSource.YAHOO;
}
- public async search(aSymbol: string): Promise<{ items: LookupItem[] }> {
+ public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
const items: LookupItem[] = [];
try {
const get = bent(
- `${this.yahooFinanceHostname}/v1/finance/search?q=${aSymbol}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
+ `${this.yahooFinanceHostname}/v1/finance/search?q=${encodeURIComponent(
+ aQuery
+ )}&lang=en-US&region=US&quotesCount=8&newsCount=0&enableFuzzyQuery=false&quotesQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=false&enableEnhancedTrivialQuery=true`,
'GET',
'json',
200
@@ -268,7 +270,7 @@ export class YahooFinanceService implements DataProviderInterface {
items.push({
symbol,
currency: value.currency,
- dataSource: DataSource.YAHOO,
+ dataSource: this.getName(),
name: value.name
});
}
diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts
index 56f8fe822b..86b0495469 100644
--- a/apps/api/src/services/interfaces/environment.interface.ts
+++ b/apps/api/src/services/interfaces/environment.interface.ts
@@ -4,6 +4,7 @@ export interface Environment extends CleanedEnvAccessors {
ACCESS_TOKEN_SALT: string;
ALPHA_VANTAGE_API_KEY: string;
CACHE_TTL: number;
+ DATA_SOURCE_PRIMARY: string;
DATA_SOURCES: string | string[]; // string is not correct, error in envalid?
ENABLE_FEATURE_BLOG: boolean;
ENABLE_FEATURE_CUSTOM_SYMBOLS: boolean;
@@ -16,6 +17,9 @@ export interface Environment extends CleanedEnvAccessors {
ENABLE_FEATURE_SYSTEM_MESSAGE: boolean;
GOOGLE_CLIENT_ID: string;
GOOGLE_SECRET: string;
+ GOOGLE_SHEETS_ACCOUNT: string;
+ GOOGLE_SHEETS_ID: string;
+ GOOGLE_SHEETS_PRIVATE_KEY: string;
JWT_SECRET_KEY: string;
MAX_ITEM_IN_CACHE: number;
MAX_ORDERS_TO_IMPORT: number;
diff --git a/package.json b/package.json
index b3de40d0e0..3d1a24d975 100644
--- a/package.json
+++ b/package.json
@@ -94,6 +94,7 @@
"cryptocurrencies": "7.0.0",
"date-fns": "2.22.1",
"envalid": "7.2.1",
+ "google-spreadsheet": "3.2.0",
"http-status-codes": "2.2.0",
"ionicons": "5.5.1",
"lodash": "4.17.21",
@@ -143,6 +144,7 @@
"@types/big.js": "6.1.2",
"@types/cache-manager": "3.4.2",
"@types/color": "3.0.2",
+ "@types/google-spreadsheet": "3.1.5",
"@types/jest": "27.0.2",
"@types/lodash": "4.14.174",
"@types/node": "14.14.33",
diff --git a/prisma/migrations/20220108083624_added_google_sheets_to_data_source/migration.sql b/prisma/migrations/20220108083624_added_google_sheets_to_data_source/migration.sql
new file mode 100644
index 0000000000..f8923af93d
--- /dev/null
+++ b/prisma/migrations/20220108083624_added_google_sheets_to_data_source/migration.sql
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "DataSource" ADD VALUE 'GOOGLE_SHEETS';
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
index b0d7a313cb..0c89f8ec55 100644
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -184,6 +184,7 @@ enum AssetSubClass {
enum DataSource {
ALPHA_VANTAGE
GHOSTFOLIO
+ GOOGLE_SHEETS
RAKUTEN
YAHOO
}
diff --git a/yarn.lock b/yarn.lock
index c4f08c7009..bce12e97b2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4487,6 +4487,11 @@
"@types/minimatch" "*"
"@types/node" "*"
+ version "3.1.5"
+ resolved "https://registry.yarnpkg.com/@types/google-spreadsheet/-/google-spreadsheet-3.1.5.tgz#2bdc6f9f5372551e0506cb6ef3f562adcf44fc2e"
+ integrity sha512-7N+mDtZ1pmya2RRFPPl4KYc2TRgiqCNBLUZfyrKfER+u751JgCO+C24/LzF70UmUm/zhHUbzRZ5mtfaxekQ1ZQ==
+
"@types/graceful-fs@^4.1.2":
version "4.1.5"
resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15"
@@ -5284,6 +5289,13 @@ abbrev@1:
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==
+abort-controller@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
+ integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==
+ dependencies:
+ event-target-shim "^5.0.0"
+
accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7:
version "1.3.7"
resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd"
@@ -5774,7 +5786,7 @@ array.prototype.map@^1.0.3:
es-array-method-boxes-properly "^1.0.0"
is-string "^1.0.5"
-arrify@^2.0.1:
+arrify@^2.0.0, arrify@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa"
integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==
@@ -5903,6 +5915,13 @@ [email protected]:
dependencies:
follow-redirects "^1.14.4"
+axios@^0.21.4:
+ version "0.21.4"
+ resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
+ integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
+ dependencies:
+ follow-redirects "^1.14.0"
+
version "2.0.2"
resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.0.2.tgz#ea187abe5b9002b377f925d8bf7d1c561adf38f9"
@@ -6124,7 +6143,7 @@ balanced-match@^1.0.0:
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
-base64-js@^1.0.2, base64-js@^1.2.0, base64-js@^1.3.1:
+base64-js@^1.0.2, base64-js@^1.2.0, base64-js@^1.3.0, base64-js@^1.3.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
@@ -6190,6 +6209,11 @@ big.js@^5.2.2:
resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328"
integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==
+bignumber.js@^9.0.0:
+ version "9.0.2"
+ resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.2.tgz#71c6c6bed38de64e24a65ebe16cfcf23ae693673"
+ integrity sha512-GAcQvbpsM0pUb0zw1EI0KhQEZ+lRwR5fYaAp3vPOYuP7aDvGy6cVN6XHLauvF8SOga2y0dcLcjt3iQDTSEliyw==
+
bignumber.js@^9.0.1:
version "9.0.1"
resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.1.tgz#8d7ba124c882bfd8e43260c67475518d0689e4e5"
@@ -8233,7 +8257,7 @@ ecc-jsbn@~0.1.1:
jsbn "~0.1.0"
safer-buffer "^2.1.0"
[email protected], ecdsa-sig-formatter@^1.0.11:
version "1.0.11"
resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf"
integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==
@@ -9064,6 +9088,11 @@ etag@~1.8.1:
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
+event-target-shim@^5.0.0:
+ version "5.0.1"
+ resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
+ integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
+
eventemitter-asyncresource@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b"
@@ -9233,7 +9262,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2:
assign-symbols "^1.0.0"
is-extendable "^1.0.1"
-extend@^3.0.0, extend@~3.0.2:
+extend@^3.0.0, extend@^3.0.2, extend@~3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==
@@ -9324,6 +9353,11 @@ [email protected]:
resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884"
integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==
+fast-text-encoding@^1.0.0:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53"
+ integrity sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig==
+
fastparse@^1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/fastparse/-/fastparse-1.1.2.tgz#91728c5a5942eced8531283c79441ee4122c35a9"
@@ -9587,6 +9621,11 @@ follow-redirects@^1.0.0:
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.3.tgz#6ada78118d8d24caee595595accdc0ac6abd022e"
integrity sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==
+follow-redirects@^1.14.0:
+ version "1.14.6"
+ resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.6.tgz#8cfb281bbc035b3c067d6cd975b0f6ade6e855cd"
+ integrity sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A==
+
follow-redirects@^1.14.4:
version "1.14.5"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.5.tgz#f09a5848981d3c772b5392309778523f8d85c381"
@@ -9831,6 +9870,25 @@ gauge@^4.0.0:
strip-ansi "^6.0.1"
wide-align "^1.1.2"
+gaxios@^4.0.0:
+ version "4.3.2"
+ resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.3.2.tgz#845827c2dc25a0213c8ab4155c7a28910f5be83f"
+ integrity sha512-T+ap6GM6UZ0c4E6yb1y/hy2UB6hTrqhglp3XfmU9qbLCGRYhLVV5aRPpC4EmoG8N8zOnkYCgoBz+ScvGAARY6Q==
+ dependencies:
+ abort-controller "^3.0.0"
+ extend "^3.0.2"
+ https-proxy-agent "^5.0.0"
+ is-stream "^2.0.0"
+ node-fetch "^2.6.1"
+
+gcp-metadata@^4.2.0:
+ version "4.3.1"
+ resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.3.1.tgz#fb205fe6a90fef2fd9c85e6ba06e5559ee1eefa9"
+ integrity sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A==
+ dependencies:
+ gaxios "^4.0.0"
+ json-bigint "^1.0.0"
+
gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2:
version "1.0.0-beta.2"
resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@@ -10086,11 +10144,51 @@ globby@^9.0.0, globby@^9.2.0:
pify "^4.0.1"
slash "^2.0.0"
+google-auth-library@^6.1.3:
+ version "6.1.6"
+ resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.1.6.tgz#deacdcdb883d9ed6bac78bb5d79a078877fdf572"
+ integrity sha512-Q+ZjUEvLQj/lrVHF/IQwRo6p3s8Nc44Zk/DALsN+ac3T4HY/g/3rrufkgtl+nZ1TW7DNAw5cTChdVp4apUXVgQ==
+ dependencies:
+ arrify "^2.0.0"
+ base64-js "^1.3.0"
+ ecdsa-sig-formatter "^1.0.11"
+ fast-text-encoding "^1.0.0"
+ gaxios "^4.0.0"
+ gcp-metadata "^4.2.0"
+ gtoken "^5.0.4"
+ jws "^4.0.0"
+ lru-cache "^6.0.0"
+
+google-p12-pem@^3.0.3:
+ version "3.1.2"
+ resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.2.tgz#c3d61c2da8e10843ff830fdb0d2059046238c1d4"
+ integrity sha512-tjf3IQIt7tWCDsa0ofDQ1qqSCNzahXDxdAGJDbruWqu3eCg5CKLYKN+hi0s6lfvzYZ1GDVr+oDF9OOWlDSdf0A==
+ dependencies:
+ node-forge "^0.10.0"
+
+ version "3.2.0"
+ resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-3.2.0.tgz#ce8aa75c15705aa950ad52b091a6fc4d33dcb329"
+ integrity sha512-z7XMaqb+26rdo8p51r5O03u8aPLAPzn5YhOXYJPcf2hdMVr0dUbIARgdkRdmGiBeoV/QoU/7VNhq1MMCLZv3kQ==
+ dependencies:
+ axios "^0.21.4"
+ google-auth-library "^6.1.3"
+ lodash "^4.17.21"
+
graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6:
version "4.2.8"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a"
integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==
+gtoken@^5.0.4:
+ version "5.3.1"
+ resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.3.1.tgz#c1c2598a826f2b5df7c6bb53d7be6cf6d50c3c78"
+ integrity sha512-yqOREjzLHcbzz1UrQoxhBtpk8KjrVhuqPE7od1K2uhyxG2BHjKZetlbLw/SPZak/QqTIQW+addS+EcjqQsZbwQ==
+ dependencies:
+ gaxios "^4.0.0"
+ google-p12-pem "^3.0.3"
+ jws "^4.0.0"
+
version "5.1.1"
resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-5.1.1.tgz#cb9bee692f87c0612b232840a873904e4c135274"
@@ -12175,6 +12273,13 @@ jsesc@~0.5.0:
resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d"
integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=
+json-bigint@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1"
+ integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==
+ dependencies:
+ bignumber.js "^9.0.0"
+
json-parse-better-errors@^1.0.1, json-parse-better-errors@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
@@ -12295,6 +12400,15 @@ jwa@^1.4.1:
ecdsa-sig-formatter "1.0.11"
safe-buffer "^5.0.1"
+jwa@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc"
+ integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==
+ dependencies:
+ buffer-equal-constant-time "1.0.1"
+ ecdsa-sig-formatter "1.0.11"
+ safe-buffer "^5.0.1"
+
jwk-to-pem@^2.0.4:
version "2.0.5"
resolved "https://registry.yarnpkg.com/jwk-to-pem/-/jwk-to-pem-2.0.5.tgz#151310bcfbcf731adc5ad9f379cbc8b395742906"
@@ -12312,6 +12426,14 @@ jws@^3.2.2:
jwa "^1.4.1"
safe-buffer "^5.0.1"
+jws@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4"
+ integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==
+ dependencies:
+ jwa "^2.0.0"
+ safe-buffer "^5.0.1"
+
version "1.4.0"
resolved "https://registry.yarnpkg.com/karma-source-map-support/-/karma-source-map-support-1.4.0.tgz#58526ceccf7e8730e56effd97a4de8d712ac0d6b"