Open platform to collect and prioritize feedback
feature: pretender crawler requests if rendergun is configured (#724)
Guilherme Oenningaa9aa94e6abb
diff --git a/app/pkg/billing/stripe_test.go b/app/pkg/billing/stripe_test.goindex d478e53b8..2518ab942 100644--- a/app/pkg/billing/stripe_test.go+++ b/app/pkg/billing/stripe_test.go@@ -100,7 +100,7 @@ func TestUpdatePaymentInfo(t *testing.T) {Expect(info.CardBrand).Equals("Visa")Expect(info.CardCountry).Equals("US")Expect(info.CardLast4).Equals("4242")- Expect(int(info.CardExpMonth)).Equals(1)+ Expect(int(info.CardExpMonth)).Equals(2)Expect(int(info.CardExpYear)).Equals(2020)Expect(info.Name).Equals("")Expect(info.AddressLine1).Equals("")@@ -153,7 +153,7 @@ func TestUpdatePaymentInfo(t *testing.T) {Expect(info.CardBrand).Equals("Visa")Expect(info.CardCountry).Equals("BR")Expect(info.CardLast4).Equals("0002")- Expect(int(info.CardExpMonth)).Equals(1)+ Expect(int(info.CardExpMonth)).Equals(2)Expect(int(info.CardExpYear)).Equals(2020)Expect(info.Name).Equals("")Expect(info.AddressLine1).Equals("")diff --git a/app/pkg/env/env.go b/app/pkg/env/env.goindex 961bba278..cdb438f80 100644--- a/app/pkg/env/env.go+++ b/app/pkg/env/env.go@@ -20,7 +20,10 @@ type config struct {HostMode string `env:"HOST_MODE,default=single"`HostDomain string `env:"HOST_DOMAIN"`JWTSecret string `env:"JWT_SECRET,required"`- Database struct {+ Rendergun struct {+ URL string `env:"RENDERGUN_URL"`+ }+ Database struct {URL string `env:"DATABASE_URL,required"`MaxIdleConns int `env:"DATABASE_MAX_IDLE_CONNS,default=2,strict"`MaxOpenConns int `env:"DATABASE_MAX_OPEN_CONNS,default=4,strict"`diff --git a/app/pkg/web/context.go b/app/pkg/web/context.goindex 9270e6e14..b149bc2b0 100644--- a/app/pkg/web/context.go+++ b/app/pkg/web/context.go@@ -4,6 +4,8 @@ import ("bytes""encoding/json""fmt"+ "io"+ "io/ioutil""net/http""net/url""strconv"@@ -304,9 +306,42 @@ func (ctx *Context) Render(code int, template string, props Props) error {return ctx.JSON(code, Map{})}- buf := new(bytes.Buffer)- ctx.engine.renderer.Render(buf, template, props, ctx)- return ctx.Blob(code, UTF8HTMLContentType, buf.Bytes())+ html := new(bytes.Buffer)+ ctx.engine.renderer.Render(html, template, props, ctx)++ if len(env.Config.Rendergun.URL) > 0 && ctx.Request.IsCrawler() {+ return ctx.prerender(code, html)+ }++ return ctx.Blob(code, UTF8HTMLContentType, html.Bytes())+}++func (ctx *Context) prerender(code int, html io.Reader) error {+ renderURL := fmt.Sprintf("%s/render?url=%s", env.Config.Rendergun.URL, ctx.Request.URL.String())+ req, _ := http.NewRequest("POST", renderURL, html)+ req.Header.Set("Content-Type", "text/html")+ req.Header.Set("x-rendergun-wait-until", "networkidle0")+ req.Header.Set("x-rendergun-abort-request", "assets\\/css\\/(common|vendor|main)\\.")++ resp, err := http.DefaultClient.Do(req)+ if err != nil {+ ctx.Logger().Error(errors.Wrap(err, "failed to execute rendergun"))+ return ctx.TryAgainLater(24 * time.Hour)+ }+ defer resp.Body.Close()+ prerendered, err := ioutil.ReadAll(resp.Body)+ if err != nil {+ ctx.Logger().Error(errors.Wrap(err, "failed to copy response from rendergun to output"))+ return ctx.TryAgainLater(24 * time.Hour)+ }+ return ctx.Blob(code, UTF8HTMLContentType, prerendered)+}++//TryAgainLater returns a service unavailable response with Retry-After header+func (ctx *Context) TryAgainLater(d time.Duration) error {+ ctx.Response.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")+ ctx.Response.Header().Set("Retry-After", fmt.Sprintf("%.0f", d.Seconds()))+ return ctx.NoContent(http.StatusServiceUnavailable)}//AddParam add a single param to route parameters listdiff --git a/app/pkg/web/context_test.go b/app/pkg/web/context_test.goindex cdbc7c413..b6c572e06 100644--- a/app/pkg/web/context_test.go+++ b/app/pkg/web/context_test.go@@ -2,10 +2,12 @@ package web_testimport ("crypto/tls"+ "net/http""net/http/httptest""net/url""strings""testing"+ "time""github.com/getfider/fider/app/models". "github.com/getfider/fider/app/pkg/assert"@@ -226,3 +228,15 @@ func TestCanonicalURL_DifferentDomain(t *testing.T) {ctx.SetCanonicalURL("page-b/abc.html")Expect(ctx.Get("Canonical-URL")).Equals(`http://feedback.theavengers.com/page-b/abc.html`)}++func TestTryAgainLater(t *testing.T) {+ RegisterT(t)++ ctx := newGetContext("http://demo.test.fider.io:3000", nil)+ err := ctx.TryAgainLater(24 * time.Hour)+ Expect(err).IsNil()+ resp := ctx.Response.(*httptest.ResponseRecorder)+ Expect(resp.Code).Equals(http.StatusServiceUnavailable)+ Expect(resp.Header().Get("Cache-Control")).Equals("no-cache, no-store, must-revalidate")+ Expect(resp.Header().Get("Retry-After")).Equals("86400")+}diff --git a/app/pkg/web/request.go b/app/pkg/web/request.goindex 7e41b3216..4dadcaf3e 100644--- a/app/pkg/web/request.go+++ b/app/pkg/web/request.go@@ -5,6 +5,7 @@ import ("net""net/http""net/url"+ "regexp""strings""github.com/getfider/fider/app/pkg/errors"@@ -98,3 +99,10 @@ func (r *Request) AddCookie(cookie *http.Cookie) {func (r *Request) IsAPI() bool {return strings.HasPrefix(r.URL.Path, "/api/")}++var crawlerRegex = regexp.MustCompile("(?i)(baidu)|(msnbot)|(bingbot)|(bingpreview)|(duckduckbot)|(googlebot)|(adsbot-google)|(mediapartners-google)|(slurp)|(yandexbot)|(yandexmetrika)|(ahrefsbot)|(twitterbot)|(slackbot)|(discordbot)|(semrushBot)|(exabot)")++// IsCrawler returns true if the request is coming from a crawler+func (r *Request) IsCrawler() bool {+ return crawlerRegex.MatchString(r.GetHeader("User-Agent"))+}diff --git a/app/pkg/web/request_test.go b/app/pkg/web/request_test.goindex 468134aa4..176a4edc2 100644--- a/app/pkg/web/request_test.go+++ b/app/pkg/web/request_test.go@@ -101,3 +101,62 @@ func TestRequest_FullURL(t *testing.T) {Expect(req.IsSecure).Equals(true)Expect(req.IsAPI()).IsTrue()}++func TestRequest_IsCrawler(t *testing.T) {+ RegisterT(t)++ var testCases = []struct {+ userAgent string+ isCrawler bool+ }{+ {"Baidu Union Search", true},+ {"msnbot/2.0b (+http://search.msn.com/msnbot.htm)", true},+ {"Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)", true},+ {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", true},+ {"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", true},+ {"Googlebot/2.1 (+http://www.googlebot.com/bot.html)", true},+ {"AdsBot-Google (+http://www.google.com/adsbot.html)", true},+ {"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", true},+ {"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", true},+ {"Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)", true},+ {"Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", true},+ {"Googlebot-News", true},+ {"Twitterbot/1.0", true},+ {"Slackbot 1.0(+https://api.slack.com/robots)", true},+ {"Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", true},+ {"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", true},+ {"SemrushBot", true},+ {"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails); Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", true},+ {"Google Chrome", false},+ {"Google", false},+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299", false},+ {"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", false},+ {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0", false},+ {"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", false},+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", false},+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134", false},+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", false},+ {"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36", false},+ {"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36", false},+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0", false},+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", false},+ {"Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1", false},+ {"Safari", false},+ {"Firefox", false},+ {"Anything", false},+ }++ for _, tt := range testCases {+ header := make(http.Header, 0)+ header.Set("User-Agent", tt.userAgent)++ req := web.WrapRequest(+ &http.Request{+ Method: "GET",+ Header: header,+ Host: "demo.test.fider.io",+ },+ )+ Expect(req.IsCrawler()).Equals(tt.isCrawler)+ }+}