Business intelligence made simple
Added experimental support for uploads
Andrew Kane955a2c13953d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 73ec214f..a0498f1b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,6 +1,7 @@## 2.3.0 (unreleased)- Added support for archiving queries+- Added experimental support for uploads## 2.2.8 (2020-11-01)diff --git a/README.md b/README.mdindex 69385f9c..aab83a2c 100644--- a/README.md+++ b/README.md@@ -27,6 +27,7 @@ Blazer is also available as a [Docker image](https://github.com/ankane/blazer-do- [Checks](#checks)- [Anomaly Detection](#anomaly-detection)- [Forecasting](#forecasting)+- [Uploads](#uploads)- [Data Sources](#data-sources)- [Query Permissions](#query-permissions)@@ -491,6 +492,30 @@ And add to `config/blazer.yml`:forecasting: trend```+## Uploads++Blazer has experimental support for creating database tables from CSV files. Run:++```sh+rails generate blazer:uploads+rails db:migrate+```++And add to `config/blazer.yml`:++```yml+uploads:+ url: postgres://...+ schema: uploads+ data_source: main+```++This feature requires PostgreSQL. Create a new schema just for uploads to ensure existing tables aren’t overwritten.++```sql+CREATE SCHEMA uploads;+```+## Data SourcesBlazer supports multiple data sources :tada:diff --git a/app/controllers/blazer/queries_controller.rb b/app/controllers/blazer/queries_controller.rbindex d8cf6555..22d76234 100644--- a/app/controllers/blazer/queries_controller.rb+++ b/app/controllers/blazer/queries_controller.rb@@ -38,6 +38,12 @@ module Blazerif params[:fork_query_id]@query.statement ||= Blazer::Query.find(params[:fork_query_id]).try(:statement)end+ if params[:upload_id]+ upload = Blazer::Upload.find(params[:upload_id])+ upload_settings = Blazer.settings["uploads"]+ @query.data_source ||= upload_settings["data_source"]+ @query.statement ||= "SELECT * FROM #{upload.table_name} LIMIT 10"+ endenddef creatediff --git a/app/controllers/blazer/uploads_controller.rb b/app/controllers/blazer/uploads_controller.rbnew file mode 100644index 00000000..b539938e--- /dev/null+++ b/app/controllers/blazer/uploads_controller.rb@@ -0,0 +1,135 @@+module Blazer+ class UploadsController < BaseController+ before_action :ensure_uploads+ before_action :set_upload, only: [:show, :edit, :update, :destroy]++ def index+ @uploads = Blazer::Upload.order(:table)+ end++ def new+ @upload = Blazer::Upload.new+ end++ def create+ @upload = Blazer::Upload.new(upload_params)+ # use creator_id instead of creator+ # since we setup association without checking if column exists+ @upload.creator = blazer_user if @upload.respond_to?(:creator_id=) && blazer_user++ success = params.require(:upload).key?(:file)+ if success+ Blazer::Upload.transaction do+ success = @upload.save+ if success+ begin+ update_file(@upload)+ rescue CSV::MalformedCSVError, Blazer::UploadError => e+ @upload.errors.add(:base, e.message)+ success = false+ raise ActiveRecord::Rollback+ end+ end+ end+ else+ @upload.errors.add(:base, "File can't be blank")+ end++ if success+ redirect_to upload_path(@upload)+ else+ render_errors @upload+ end+ end++ def show+ redirect_to new_query_path(upload_id: @upload.id)+ end++ def edit+ end++ def update+ original_table = @upload.table+ @upload.assign_attributes(upload_params)++ success = nil+ Blazer::Upload.transaction do+ success = @upload.save+ if success+ if params.require(:upload).key?(:file)+ begin+ update_file(@upload, drop: original_table)+ rescue CSV::MalformedCSVError, Blazer::UploadError => e+ @upload.errors.add(:base, e.message)+ success = false+ raise ActiveRecord::Rollback+ end+ elsif @upload.table != original_table+ Blazer.uploads_connection.execute("ALTER TABLE #{Blazer.uploads_table_name(original_table)} RENAME TO #{Blazer.uploads_connection.quote_table_name(@upload.table)}")+ end+ end+ end++ if success+ redirect_to upload_path(@upload)+ else+ render_errors @upload+ end+ end++ def destroy+ Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{@upload.table_name}")+ @upload.destroy+ redirect_to uploads_path+ end++ private++ def update_file(upload, drop: nil)+ file = params.require(:upload).fetch(:file)+ raise Blazer::UploadError, "File is not a CSV" if file.content_type != "text/csv"++ contents = file.read+ rows = CSV.parse(contents, converters: %i[numeric date date_time])+ columns = rows.shift.map(&:to_s)+ column_types =+ columns.size.times.map do |i|+ values = rows.map { |r| r[i] }.uniq.compact+ if values.all? { |v| v.is_a?(Integer) && v >= -9223372036854775808 && v <= 9223372036854775807 }+ "bigint"+ elsif values.all? { |v| v.is_a?(Numeric) }+ "decimal"+ elsif values.all? { |v| v.is_a?(DateTime) }+ "timestamptz"+ elsif values.all? { |v| v.is_a?(Date) }+ "date"+ else+ "text"+ end+ end++ # maybe SET LOCAL statement_timeout = '30s'+ Blazer.uploads_connection.transaction do+ Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{Blazer.uploads_table_name(drop)}") if drop+ Blazer.uploads_connection.execute("CREATE TABLE #{upload.table_name} (#{columns.map.with_index { |c, i| "#{Blazer.uploads_connection.quote_column_name(c)} #{column_types[i]}" }.join(", ")})")+ Blazer.uploads_connection.raw_connection.copy_data("COPY #{upload.table_name} FROM STDIN CSV HEADER") do+ Blazer.uploads_connection.raw_connection.put_copy_data(contents)+ end+ end+ end++ def upload_params+ params.require(:upload).except(:file).permit(:table, :description)+ end++ def set_upload+ @upload = Blazer::Upload.find(params[:id])+ end++ # routes aren't added, but also check here+ def ensure_uploads+ render plain: "Uploads not enabled" unless Blazer.uploads?+ end+ end+enddiff --git a/app/models/blazer/upload.rb b/app/models/blazer/upload.rbnew file mode 100644index 00000000..445fadab--- /dev/null+++ b/app/models/blazer/upload.rb@@ -0,0 +1,11 @@+module Blazer+ class Upload < Record+ belongs_to :creator, optional: true, class_name: Blazer.user_class.to_s if Blazer.user_class++ validates :table, presence: true, uniqueness: true, format: {with: /\A[a-z0-9_]+\z/, message: "can only contain lowercase letters, numbers, and underscores"}, length: {maximum: 63}++ def table_name+ Blazer.uploads_table_name(table)+ end+ end+enddiff --git a/app/models/blazer/uploads_connection.rb b/app/models/blazer/uploads_connection.rbnew file mode 100644index 00000000..bac18d04--- /dev/null+++ b/app/models/blazer/uploads_connection.rb@@ -0,0 +1,7 @@+module Blazer+ class UploadsConnection < ActiveRecord::Base+ self.abstract_class = true++ establish_connection Blazer.settings["uploads"]["url"] if Blazer.uploads?+ end+enddiff --git a/app/views/blazer/queries/home.html.erb b/app/views/blazer/queries/home.html.erbindex 33d66269..18f088c3 100644--- a/app/views/blazer/queries/home.html.erb+++ b/app/views/blazer/queries/home.html.erb@@ -19,6 +19,9 @@</button><ul class="dropdown-menu"><li><%= link_to "Checks", checks_path %></li>+ <% if Blazer.uploads? %>+ <li><%= link_to "Uploads", uploads_path %></li>+ <% end %><li role="separator" class="divider"></li><li><%= link_to "New Dashboard", new_dashboard_path %></li><li><%= link_to "New Check", new_check_path %></li>diff --git a/app/views/blazer/uploads/_form.html.erb b/app/views/blazer/uploads/_form.html.erbnew file mode 100644index 00000000..ddb4caa6--- /dev/null+++ b/app/views/blazer/uploads/_form.html.erb@@ -0,0 +1,27 @@+<%= form_for @upload, html: {class: "small-form"} do |f| %>+ <% if @upload.errors.any? %>+ <div class="alert alert-danger"><%= @upload.errors.full_messages.first %></div>+ <% elsif [email protected]? %>+ <p>Create a database table from a CSV file. The table will be created in the <code><%= Blazer.settings["uploads"]["schema"] %></code> schema.</p>+ <% end %>++ <div class="form-group">+ <%= f.label :table %>+ <%= f.text_field :table, class: "form-control" %>+ </div>+ <div class="form-group">+ <%= f.label :description %>+ <%= f.text_area :description, placeholder: "Optional", style: "height: 60px;", class: "form-control" %>+ </div>+ <div class="form-group">+ <%= f.label :file %>+ <%= f.file_field :file, accept: "text/csv", style: "margin-top: 6px; margin-bottom: 21px;" %>+ </div>+ <p>+ <% if @upload.persisted? %>+ <%= link_to "Delete", upload_path(@upload), method: :delete, "data-confirm" => "Are you sure?", class: "btn btn-danger" %>+ <% end %>+ <%= f.submit "Save", class: "btn btn-success" %>+ <%= link_to "Back", :back, class: "btn btn-link" %>+ </p>+<% end %>diff --git a/app/views/blazer/uploads/edit.html.erb b/app/views/blazer/uploads/edit.html.erbnew file mode 100644index 00000000..156ab824--- /dev/null+++ b/app/views/blazer/uploads/edit.html.erb@@ -0,0 +1,3 @@+<% blazer_title "Edit Upload" %>++<%= render partial: "form" %>diff --git a/app/views/blazer/uploads/index.html.erb b/app/views/blazer/uploads/index.html.erbnew file mode 100644index 00000000..48f4884a--- /dev/null+++ b/app/views/blazer/uploads/index.html.erb@@ -0,0 +1,54 @@+<% blazer_title "Uploads" %>++<div id="header">+ <div class="pull-right" style="line-height: 34px;">+ <div class="btn-group">+ <%= link_to "New Upload", new_upload_path, class: "btn btn-info" %>+ <button type="button" class="btn btn-info dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">+ <span class="caret"></span>+ <span class="sr-only">Toggle Dropdown</span>+ </button>+ <ul class="dropdown-menu">+ <li><%= link_to "Home", root_path %></li>+ <li role="separator" class="divider"></li>+ <li><%= link_to "New Query", new_query_path %></li>+ <li><%= link_to "New Dashboard", new_dashboard_path %></li>+ <li><%= link_to "New Check", new_check_path %></li>+ </ul>+ </div>+ </div>++ <input id="search" type="text" placeholder="Start typing a table or person" style="width: 300px; display: inline-block;" class="search form-control" />+</div>++<table id="uploads" class="table">+ <thead>+ <tr>+ <th>Table</th>+ <th style="width: 60%;"></th>+ <% if Blazer.user_class %>+ <th style="width: 20%; text-align: right;">Mastermind</th>+ <% end%>+ </tr>+ </thead>+ <tbody>+ <% @uploads.each do |upload| %>+ <tr>+ <td><%= link_to upload.table, edit_upload_path(upload) %></td>+ <td><%= truncate(upload.description, length: 100, separator: " ") %></td>+ <% if Blazer.user_class %>+ <td class="creator"><%= blazer_user && upload.creator == blazer_user ? "You" : upload.creator.try(Blazer.user_name) %></td>+ <% end %>+ </tr>+ <% end %>+ </tbody>+</table>++<script>+ $("#search").on("keyup", function() {+ var value = $(this).val().toLowerCase()+ $("#uploads tbody tr").filter( function() {+ $(this).toggle($(this).text().toLowerCase().indexOf(value) > -1)+ })+ }).focus()+</script>diff --git a/app/views/blazer/uploads/new.html.erb b/app/views/blazer/uploads/new.html.erbnew file mode 100644index 00000000..829244a3--- /dev/null+++ b/app/views/blazer/uploads/new.html.erb@@ -0,0 +1,3 @@+<% blazer_title "New Upload" %>++<%= render partial: "form" %>diff --git a/config/routes.rb b/config/routes.rbindex b9c94bb7..5df190d9 100644--- a/config/routes.rb+++ b/config/routes.rb@@ -16,5 +16,10 @@ Blazer::Engine.routes.draw dopost :refresh, on: :memberend+ if Blazer.uploads?+ resources :uploads do+ end+ end+root to: "queries#home"enddiff --git a/lib/blazer.rb b/lib/blazer.rbindex 2bdbce36..bef51238 100644--- a/lib/blazer.rb+++ b/lib/blazer.rb@@ -32,6 +32,7 @@ require "blazer/engine"module Blazerclass Error < StandardError; end+ class UploadError < Error; endclass TimeoutNotSupported < Error; endclass << self@@ -206,6 +207,23 @@ module Blazerslack_webhook_url.present?end+ def self.uploads?+ settings.key?("uploads")+ end++ def self.uploads_connection+ raise "Empty url for uploads" unless settings.dig("uploads", "url")+ Blazer::UploadsConnection.connection+ end++ def self.uploads_schema+ settings.dig("uploads", "schema") || "uploads"+ end++ def self.uploads_table_name(name)+ uploads_connection.quote_table_name("#{uploads_schema}.#{name}")+ end+def self.adapters@adapters ||= {}enddiff --git a/lib/generators/blazer/templates/config.yml.tt b/lib/generators/blazer/templates/config.yml.ttindex 8ee8dd96..65230e65 100644--- a/lib/generators/blazer/templates/config.yml.tt+++ b/lib/generators/blazer/templates/config.yml.tt@@ -71,3 +71,9 @@ check_schedules:# enable map# mapbox_access_token: <%%= ENV["MAPBOX_ACCESS_TOKEN"] %>++# enable uploads+# uploads:+# url: <%%= ENV["BLAZER_UPLOADS_URL"] %>+# schema: uploads+# data_source: maindiff --git a/lib/generators/blazer/templates/uploads.rb.tt b/lib/generators/blazer/templates/uploads.rb.ttnew file mode 100644index 00000000..6b5f27ff--- /dev/null+++ b/lib/generators/blazer/templates/uploads.rb.tt@@ -0,0 +1,10 @@+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>+ def change+ create_table :blazer_uploads do |t|+ t.references :creator+ t.string :table+ t.text :description+ t.timestamps null: false+ end+ end+enddiff --git a/lib/generators/blazer/uploads_generator.rb b/lib/generators/blazer/uploads_generator.rbnew file mode 100644index 00000000..53d111ac--- /dev/null+++ b/lib/generators/blazer/uploads_generator.rb@@ -0,0 +1,18 @@+require "rails/generators/active_record"++module Blazer+ module Generators+ class UploadsGenerator < Rails::Generators::Base+ include ActiveRecord::Generators::Migration+ source_root File.join(__dir__, "templates")++ def copy_migration+ migration_template "uploads.rb", "db/migrate/create_blazer_uploads.rb", migration_version: migration_version+ end++ def migration_version+ "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"+ end+ end+ end+end