Skip to content

Business intelligence made simple

Added experimental support for uploads

Andrew Kane955a2c13953d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73ec214f..a0498f1b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,7 @@
## 2.3.0 (unreleased)
- Added support for archiving queries
+- Added experimental support for uploads
## 2.2.8 (2020-11-01)
diff --git a/README.md b/README.md
index 69385f9c..aab83a2c 100644
--- a/README.md
+++ b/README.md
@@ -27,6 +27,7 @@ Blazer is also available as a [Docker image](https://github.com/ankane/blazer-do
- [Checks](#checks)
- [Anomaly Detection](#anomaly-detection)
- [Forecasting](#forecasting)
+- [Uploads](#uploads)
- [Data Sources](#data-sources)
- [Query Permissions](#query-permissions)
@@ -491,6 +492,30 @@ And add to `config/blazer.yml`:
forecasting: trend
```
+## Uploads
+
+Blazer has experimental support for creating database tables from CSV files. Run:
+
+```sh
+rails generate blazer:uploads
+rails db:migrate
+```
+
+And add to `config/blazer.yml`:
+
+```yml
+uploads:
+ url: postgres://...
+ schema: uploads
+ data_source: main
+```
+
+This feature requires PostgreSQL. Create a new schema just for uploads to ensure existing tables aren’t overwritten.
+
+```sql
+CREATE SCHEMA uploads;
+```
+
## Data Sources
Blazer supports multiple data sources :tada:
diff --git a/app/controllers/blazer/queries_controller.rb b/app/controllers/blazer/queries_controller.rb
index d8cf6555..22d76234 100644
--- a/app/controllers/blazer/queries_controller.rb
+++ b/app/controllers/blazer/queries_controller.rb
@@ -38,6 +38,12 @@ module Blazer
if params[:fork_query_id]
@query.statement ||= Blazer::Query.find(params[:fork_query_id]).try(:statement)
end
+ if params[:upload_id]
+ upload = Blazer::Upload.find(params[:upload_id])
+ upload_settings = Blazer.settings["uploads"]
+ @query.data_source ||= upload_settings["data_source"]
+ @query.statement ||= "SELECT * FROM #{upload.table_name} LIMIT 10"
+ end
end
def create
diff --git a/app/controllers/blazer/uploads_controller.rb b/app/controllers/blazer/uploads_controller.rb
new file mode 100644
index 00000000..b539938e
--- /dev/null
+++ b/app/controllers/blazer/uploads_controller.rb
@@ -0,0 +1,135 @@
+module Blazer
+ class UploadsController < BaseController
+ before_action :ensure_uploads
+ before_action :set_upload, only: [:show, :edit, :update, :destroy]
+
+ def index
+ @uploads = Blazer::Upload.order(:table)
+ end
+
+ def new
+ @upload = Blazer::Upload.new
+ end
+
+ def create
+ @upload = Blazer::Upload.new(upload_params)
+ # use creator_id instead of creator
+ # since we setup association without checking if column exists
+ @upload.creator = blazer_user if @upload.respond_to?(:creator_id=) && blazer_user
+
+ success = params.require(:upload).key?(:file)
+ if success
+ Blazer::Upload.transaction do
+ success = @upload.save
+ if success
+ begin
+ update_file(@upload)
+ rescue CSV::MalformedCSVError, Blazer::UploadError => e
+ @upload.errors.add(:base, e.message)
+ success = false
+ raise ActiveRecord::Rollback
+ end
+ end
+ end
+ else
+ @upload.errors.add(:base, "File can't be blank")
+ end
+
+ if success
+ redirect_to upload_path(@upload)
+ else
+ render_errors @upload
+ end
+ end
+
+ def show
+ redirect_to new_query_path(upload_id: @upload.id)
+ end
+
+ def edit
+ end
+
+ def update
+ original_table = @upload.table
+ @upload.assign_attributes(upload_params)
+
+ success = nil
+ Blazer::Upload.transaction do
+ success = @upload.save
+ if success
+ if params.require(:upload).key?(:file)
+ begin
+ update_file(@upload, drop: original_table)
+ rescue CSV::MalformedCSVError, Blazer::UploadError => e
+ @upload.errors.add(:base, e.message)
+ success = false
+ raise ActiveRecord::Rollback
+ end
+ elsif @upload.table != original_table
+ Blazer.uploads_connection.execute("ALTER TABLE #{Blazer.uploads_table_name(original_table)} RENAME TO #{Blazer.uploads_connection.quote_table_name(@upload.table)}")
+ end
+ end
+ end
+
+ if success
+ redirect_to upload_path(@upload)
+ else
+ render_errors @upload
+ end
+ end
+
+ def destroy
+ Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{@upload.table_name}")
+ @upload.destroy
+ redirect_to uploads_path
+ end
+
+ private
+
+ def update_file(upload, drop: nil)
+ file = params.require(:upload).fetch(:file)
+ raise Blazer::UploadError, "File is not a CSV" if file.content_type != "text/csv"
+
+ contents = file.read
+ rows = CSV.parse(contents, converters: %i[numeric date date_time])
+ columns = rows.shift.map(&:to_s)
+ column_types =
+ columns.size.times.map do |i|
+ values = rows.map { |r| r[i] }.uniq.compact
+ if values.all? { |v| v.is_a?(Integer) && v >= -9223372036854775808 && v <= 9223372036854775807 }
+ "bigint"
+ elsif values.all? { |v| v.is_a?(Numeric) }
+ "decimal"
+ elsif values.all? { |v| v.is_a?(DateTime) }
+ "timestamptz"
+ elsif values.all? { |v| v.is_a?(Date) }
+ "date"
+ else
+ "text"
+ end
+ end
+
+ # maybe SET LOCAL statement_timeout = '30s'
+ Blazer.uploads_connection.transaction do
+ Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{Blazer.uploads_table_name(drop)}") if drop
+ Blazer.uploads_connection.execute("CREATE TABLE #{upload.table_name} (#{columns.map.with_index { |c, i| "#{Blazer.uploads_connection.quote_column_name(c)} #{column_types[i]}" }.join(", ")})")
+ Blazer.uploads_connection.raw_connection.copy_data("COPY #{upload.table_name} FROM STDIN CSV HEADER") do
+ Blazer.uploads_connection.raw_connection.put_copy_data(contents)
+ end
+ end
+ end
+
+ def upload_params
+ params.require(:upload).except(:file).permit(:table, :description)
+ end
+
+ def set_upload
+ @upload = Blazer::Upload.find(params[:id])
+ end
+
+ # routes aren't added, but also check here
+ def ensure_uploads
+ render plain: "Uploads not enabled" unless Blazer.uploads?
+ end
+ end
+end
diff --git a/app/models/blazer/upload.rb b/app/models/blazer/upload.rb
new file mode 100644
index 00000000..445fadab
--- /dev/null
+++ b/app/models/blazer/upload.rb
@@ -0,0 +1,11 @@
+module Blazer
+ class Upload < Record
+ belongs_to :creator, optional: true, class_name: Blazer.user_class.to_s if Blazer.user_class
+
+ validates :table, presence: true, uniqueness: true, format: {with: /\A[a-z0-9_]+\z/, message: "can only contain lowercase letters, numbers, and underscores"}, length: {maximum: 63}
+
+ def table_name
+ Blazer.uploads_table_name(table)
+ end
+ end
+end
diff --git a/app/models/blazer/uploads_connection.rb b/app/models/blazer/uploads_connection.rb
new file mode 100644
index 00000000..bac18d04
--- /dev/null
+++ b/app/models/blazer/uploads_connection.rb
@@ -0,0 +1,7 @@
+module Blazer
+ class UploadsConnection < ActiveRecord::Base
+ self.abstract_class = true
+
+ establish_connection Blazer.settings["uploads"]["url"] if Blazer.uploads?
+ end
+end
diff --git a/app/views/blazer/queries/home.html.erb b/app/views/blazer/queries/home.html.erb
index 33d66269..18f088c3 100644
--- a/app/views/blazer/queries/home.html.erb
+++ b/app/views/blazer/queries/home.html.erb
@@ -19,6 +19,9 @@
</button>
<ul class="dropdown-menu">
<li><%= link_to "Checks", checks_path %></li>
+ <% if Blazer.uploads? %>
+ <li><%= link_to "Uploads", uploads_path %></li>
+ <% end %>
<li role="separator" class="divider"></li>
<li><%= link_to "New Dashboard", new_dashboard_path %></li>
<li><%= link_to "New Check", new_check_path %></li>
diff --git a/app/views/blazer/uploads/_form.html.erb b/app/views/blazer/uploads/_form.html.erb
new file mode 100644
index 00000000..ddb4caa6
--- /dev/null
+++ b/app/views/blazer/uploads/_form.html.erb
@@ -0,0 +1,27 @@
+<%= form_for @upload, html: {class: "small-form"} do |f| %>
+ <% if @upload.errors.any? %>
+ <div class="alert alert-danger"><%= @upload.errors.full_messages.first %></div>
+ <% elsif [email protected]? %>
+ <p>Create a database table from a CSV file. The table will be created in the <code><%= Blazer.settings["uploads"]["schema"] %></code> schema.</p>
+ <% end %>
+
+ <div class="form-group">
+ <%= f.label :table %>
+ <%= f.text_field :table, class: "form-control" %>
+ </div>
+ <div class="form-group">
+ <%= f.label :description %>
+ <%= f.text_area :description, placeholder: "Optional", style: "height: 60px;", class: "form-control" %>
+ </div>
+ <div class="form-group">
+ <%= f.label :file %>
+ <%= f.file_field :file, accept: "text/csv", style: "margin-top: 6px; margin-bottom: 21px;" %>
+ </div>
+ <p>
+ <% if @upload.persisted? %>
+ <%= link_to "Delete", upload_path(@upload), method: :delete, "data-confirm" => "Are you sure?", class: "btn btn-danger" %>
+ <% end %>
+ <%= f.submit "Save", class: "btn btn-success" %>
+ <%= link_to "Back", :back, class: "btn btn-link" %>
+ </p>
+<% end %>
diff --git a/app/views/blazer/uploads/edit.html.erb b/app/views/blazer/uploads/edit.html.erb
new file mode 100644
index 00000000..156ab824
--- /dev/null
+++ b/app/views/blazer/uploads/edit.html.erb
@@ -0,0 +1,3 @@
+<% blazer_title "Edit Upload" %>
+
+<%= render partial: "form" %>
diff --git a/app/views/blazer/uploads/index.html.erb b/app/views/blazer/uploads/index.html.erb
new file mode 100644
index 00000000..48f4884a
--- /dev/null
+++ b/app/views/blazer/uploads/index.html.erb
@@ -0,0 +1,54 @@
+<% blazer_title "Uploads" %>
+
+<div id="header">
+ <div class="pull-right" style="line-height: 34px;">
+ <div class="btn-group">
+ <%= link_to "New Upload", new_upload_path, class: "btn btn-info" %>
+ <button type="button" class="btn btn-info dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+ <span class="caret"></span>
+ <span class="sr-only">Toggle Dropdown</span>
+ </button>
+ <ul class="dropdown-menu">
+ <li><%= link_to "Home", root_path %></li>
+ <li role="separator" class="divider"></li>
+ <li><%= link_to "New Query", new_query_path %></li>
+ <li><%= link_to "New Dashboard", new_dashboard_path %></li>
+ <li><%= link_to "New Check", new_check_path %></li>
+ </ul>
+ </div>
+ </div>
+
+ <input id="search" type="text" placeholder="Start typing a table or person" style="width: 300px; display: inline-block;" class="search form-control" />
+</div>
+
+<table id="uploads" class="table">
+ <thead>
+ <tr>
+ <th>Table</th>
+ <th style="width: 60%;"></th>
+ <% if Blazer.user_class %>
+ <th style="width: 20%; text-align: right;">Mastermind</th>
+ <% end%>
+ </tr>
+ </thead>
+ <tbody>
+ <% @uploads.each do |upload| %>
+ <tr>
+ <td><%= link_to upload.table, edit_upload_path(upload) %></td>
+ <td><%= truncate(upload.description, length: 100, separator: " ") %></td>
+ <% if Blazer.user_class %>
+ <td class="creator"><%= blazer_user && upload.creator == blazer_user ? "You" : upload.creator.try(Blazer.user_name) %></td>
+ <% end %>
+ </tr>
+ <% end %>
+ </tbody>
+</table>
+
+<script>
+ $("#search").on("keyup", function() {
+ var value = $(this).val().toLowerCase()
+ $("#uploads tbody tr").filter( function() {
+ $(this).toggle($(this).text().toLowerCase().indexOf(value) > -1)
+ })
+ }).focus()
+</script>
diff --git a/app/views/blazer/uploads/new.html.erb b/app/views/blazer/uploads/new.html.erb
new file mode 100644
index 00000000..829244a3
--- /dev/null
+++ b/app/views/blazer/uploads/new.html.erb
@@ -0,0 +1,3 @@
+<% blazer_title "New Upload" %>
+
+<%= render partial: "form" %>
diff --git a/config/routes.rb b/config/routes.rb
index b9c94bb7..5df190d9 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -16,5 +16,10 @@ Blazer::Engine.routes.draw do
post :refresh, on: :member
end
+ if Blazer.uploads?
+ resources :uploads do
+ end
+ end
+
root to: "queries#home"
end
diff --git a/lib/blazer.rb b/lib/blazer.rb
index 2bdbce36..bef51238 100644
--- a/lib/blazer.rb
+++ b/lib/blazer.rb
@@ -32,6 +32,7 @@ require "blazer/engine"
module Blazer
class Error < StandardError; end
+ class UploadError < Error; end
class TimeoutNotSupported < Error; end
class << self
@@ -206,6 +207,23 @@ module Blazer
slack_webhook_url.present?
end
+ def self.uploads?
+ settings.key?("uploads")
+ end
+
+ def self.uploads_connection
+ raise "Empty url for uploads" unless settings.dig("uploads", "url")
+ Blazer::UploadsConnection.connection
+ end
+
+ def self.uploads_schema
+ settings.dig("uploads", "schema") || "uploads"
+ end
+
+ def self.uploads_table_name(name)
+ uploads_connection.quote_table_name("#{uploads_schema}.#{name}")
+ end
+
def self.adapters
@adapters ||= {}
end
diff --git a/lib/generators/blazer/templates/config.yml.tt b/lib/generators/blazer/templates/config.yml.tt
index 8ee8dd96..65230e65 100644
--- a/lib/generators/blazer/templates/config.yml.tt
+++ b/lib/generators/blazer/templates/config.yml.tt
@@ -71,3 +71,9 @@ check_schedules:
# enable map
# mapbox_access_token: <%%= ENV["MAPBOX_ACCESS_TOKEN"] %>
+
+# enable uploads
+# uploads:
+# url: <%%= ENV["BLAZER_UPLOADS_URL"] %>
+# schema: uploads
+# data_source: main
diff --git a/lib/generators/blazer/templates/uploads.rb.tt b/lib/generators/blazer/templates/uploads.rb.tt
new file mode 100644
index 00000000..6b5f27ff
--- /dev/null
+++ b/lib/generators/blazer/templates/uploads.rb.tt
@@ -0,0 +1,10 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+ def change
+ create_table :blazer_uploads do |t|
+ t.references :creator
+ t.string :table
+ t.text :description
+ t.timestamps null: false
+ end
+ end
+end
diff --git a/lib/generators/blazer/uploads_generator.rb b/lib/generators/blazer/uploads_generator.rb
new file mode 100644
index 00000000..53d111ac
--- /dev/null
+++ b/lib/generators/blazer/uploads_generator.rb
@@ -0,0 +1,18 @@
+require "rails/generators/active_record"
+
+module Blazer
+ module Generators
+ class UploadsGenerator < Rails::Generators::Base
+ include ActiveRecord::Generators::Migration
+ source_root File.join(__dir__, "templates")
+
+ def copy_migration
+ migration_template "uploads.rb", "db/migrate/create_blazer_uploads.rb", migration_version: migration_version
+ end
+
+ def migration_version
+ "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+ end
+ end
+ end
+end