Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

feature: pretender crawler requests if rendergun is configured (#724)

Guilherme Oenningaa9aa94e6abb
diff --git a/app/pkg/billing/stripe_test.go b/app/pkg/billing/stripe_test.go
index d478e53b8..2518ab942 100644
--- a/app/pkg/billing/stripe_test.go
+++ b/app/pkg/billing/stripe_test.go
@@ -100,7 +100,7 @@ func TestUpdatePaymentInfo(t *testing.T) {
Expect(info.CardBrand).Equals("Visa")
Expect(info.CardCountry).Equals("US")
Expect(info.CardLast4).Equals("4242")
- Expect(int(info.CardExpMonth)).Equals(1)
+ Expect(int(info.CardExpMonth)).Equals(2)
Expect(int(info.CardExpYear)).Equals(2020)
Expect(info.Name).Equals("")
Expect(info.AddressLine1).Equals("")
@@ -153,7 +153,7 @@ func TestUpdatePaymentInfo(t *testing.T) {
Expect(info.CardBrand).Equals("Visa")
Expect(info.CardCountry).Equals("BR")
Expect(info.CardLast4).Equals("0002")
- Expect(int(info.CardExpMonth)).Equals(1)
+ Expect(int(info.CardExpMonth)).Equals(2)
Expect(int(info.CardExpYear)).Equals(2020)
Expect(info.Name).Equals("")
Expect(info.AddressLine1).Equals("")
diff --git a/app/pkg/env/env.go b/app/pkg/env/env.go
index 961bba278..cdb438f80 100644
--- a/app/pkg/env/env.go
+++ b/app/pkg/env/env.go
@@ -20,7 +20,10 @@ type config struct {
HostMode string `env:"HOST_MODE,default=single"`
HostDomain string `env:"HOST_DOMAIN"`
JWTSecret string `env:"JWT_SECRET,required"`
- Database struct {
+ Rendergun struct {
+ URL string `env:"RENDERGUN_URL"`
+ }
+ Database struct {
URL string `env:"DATABASE_URL,required"`
MaxIdleConns int `env:"DATABASE_MAX_IDLE_CONNS,default=2,strict"`
MaxOpenConns int `env:"DATABASE_MAX_OPEN_CONNS,default=4,strict"`
diff --git a/app/pkg/web/context.go b/app/pkg/web/context.go
index 9270e6e14..b149bc2b0 100644
--- a/app/pkg/web/context.go
+++ b/app/pkg/web/context.go
@@ -4,6 +4,8 @@ import (
"bytes"
"encoding/json"
"fmt"
+ "io"
+ "io/ioutil"
"net/http"
"net/url"
"strconv"
@@ -304,9 +306,42 @@ func (ctx *Context) Render(code int, template string, props Props) error {
return ctx.JSON(code, Map{})
}
- buf := new(bytes.Buffer)
- ctx.engine.renderer.Render(buf, template, props, ctx)
- return ctx.Blob(code, UTF8HTMLContentType, buf.Bytes())
+ html := new(bytes.Buffer)
+ ctx.engine.renderer.Render(html, template, props, ctx)
+
+ if len(env.Config.Rendergun.URL) > 0 && ctx.Request.IsCrawler() {
+ return ctx.prerender(code, html)
+ }
+
+ return ctx.Blob(code, UTF8HTMLContentType, html.Bytes())
+}
+
+func (ctx *Context) prerender(code int, html io.Reader) error {
+ renderURL := fmt.Sprintf("%s/render?url=%s", env.Config.Rendergun.URL, ctx.Request.URL.String())
+ req, _ := http.NewRequest("POST", renderURL, html)
+ req.Header.Set("Content-Type", "text/html")
+ req.Header.Set("x-rendergun-wait-until", "networkidle0")
+ req.Header.Set("x-rendergun-abort-request", "assets\\/css\\/(common|vendor|main)\\.")
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ ctx.Logger().Error(errors.Wrap(err, "failed to execute rendergun"))
+ return ctx.TryAgainLater(24 * time.Hour)
+ }
+ defer resp.Body.Close()
+ prerendered, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ ctx.Logger().Error(errors.Wrap(err, "failed to copy response from rendergun to output"))
+ return ctx.TryAgainLater(24 * time.Hour)
+ }
+ return ctx.Blob(code, UTF8HTMLContentType, prerendered)
+}
+
+//TryAgainLater returns a service unavailable response with Retry-After header
+func (ctx *Context) TryAgainLater(d time.Duration) error {
+ ctx.Response.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+ ctx.Response.Header().Set("Retry-After", fmt.Sprintf("%.0f", d.Seconds()))
+ return ctx.NoContent(http.StatusServiceUnavailable)
}
//AddParam add a single param to route parameters list
diff --git a/app/pkg/web/context_test.go b/app/pkg/web/context_test.go
index cdbc7c413..b6c572e06 100644
--- a/app/pkg/web/context_test.go
+++ b/app/pkg/web/context_test.go
@@ -2,10 +2,12 @@ package web_test
import (
"crypto/tls"
+ "net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
+ "time"
"github.com/getfider/fider/app/models"
. "github.com/getfider/fider/app/pkg/assert"
@@ -226,3 +228,15 @@ func TestCanonicalURL_DifferentDomain(t *testing.T) {
ctx.SetCanonicalURL("page-b/abc.html")
Expect(ctx.Get("Canonical-URL")).Equals(`http://feedback.theavengers.com/page-b/abc.html`)
}
+
+func TestTryAgainLater(t *testing.T) {
+ RegisterT(t)
+
+ ctx := newGetContext("http://demo.test.fider.io:3000", nil)
+ err := ctx.TryAgainLater(24 * time.Hour)
+ Expect(err).IsNil()
+ resp := ctx.Response.(*httptest.ResponseRecorder)
+ Expect(resp.Code).Equals(http.StatusServiceUnavailable)
+ Expect(resp.Header().Get("Cache-Control")).Equals("no-cache, no-store, must-revalidate")
+ Expect(resp.Header().Get("Retry-After")).Equals("86400")
+}
diff --git a/app/pkg/web/request.go b/app/pkg/web/request.go
index 7e41b3216..4dadcaf3e 100644
--- a/app/pkg/web/request.go
+++ b/app/pkg/web/request.go
@@ -5,6 +5,7 @@ import (
"net"
"net/http"
"net/url"
+ "regexp"
"strings"
"github.com/getfider/fider/app/pkg/errors"
@@ -98,3 +99,10 @@ func (r *Request) AddCookie(cookie *http.Cookie) {
func (r *Request) IsAPI() bool {
return strings.HasPrefix(r.URL.Path, "/api/")
}
+
+var crawlerRegex = regexp.MustCompile("(?i)(baidu)|(msnbot)|(bingbot)|(bingpreview)|(duckduckbot)|(googlebot)|(adsbot-google)|(mediapartners-google)|(slurp)|(yandexbot)|(yandexmetrika)|(ahrefsbot)|(twitterbot)|(slackbot)|(discordbot)|(semrushBot)|(exabot)")
+
+// IsCrawler returns true if the request is coming from a crawler
+func (r *Request) IsCrawler() bool {
+ return crawlerRegex.MatchString(r.GetHeader("User-Agent"))
+}
diff --git a/app/pkg/web/request_test.go b/app/pkg/web/request_test.go
index 468134aa4..176a4edc2 100644
--- a/app/pkg/web/request_test.go
+++ b/app/pkg/web/request_test.go
@@ -101,3 +101,62 @@ func TestRequest_FullURL(t *testing.T) {
Expect(req.IsSecure).Equals(true)
Expect(req.IsAPI()).IsTrue()
}
+
+func TestRequest_IsCrawler(t *testing.T) {
+ RegisterT(t)
+
+ var testCases = []struct {
+ userAgent string
+ isCrawler bool
+ }{
+ {"Baidu Union Search", true},
+ {"msnbot/2.0b (+http://search.msn.com/msnbot.htm)", true},
+ {"Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)", true},
+ {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", true},
+ {"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", true},
+ {"Googlebot/2.1 (+http://www.googlebot.com/bot.html)", true},
+ {"AdsBot-Google (+http://www.google.com/adsbot.html)", true},
+ {"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", true},
+ {"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", true},
+ {"Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)", true},
+ {"Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", true},
+ {"Googlebot-News", true},
+ {"Twitterbot/1.0", true},
+ {"Slackbot 1.0(+https://api.slack.com/robots)", true},
+ {"Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", true},
+ {"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", true},
+ {"SemrushBot", true},
+ {"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails); Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", true},
+ {"Google Chrome", false},
+ {"Google", false},
+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299", false},
+ {"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", false},
+ {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0", false},
+ {"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", false},
+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", false},
+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134", false},
+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", false},
+ {"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36", false},
+ {"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36", false},
+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0", false},
+ {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", false},
+ {"Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1", false},
+ {"Safari", false},
+ {"Firefox", false},
+ {"Anything", false},
+ }
+
+ for _, tt := range testCases {
+ header := make(http.Header, 0)
+ header.Set("User-Agent", tt.userAgent)
+
+ req := web.WrapRequest(
+ &http.Request{
+ Method: "GET",
+ Header: header,
+ Host: "demo.test.fider.io",
+ },
+ )
+ Expect(req.IsCrawler()).Equals(tt.isCrawler)
+ }
+}