diff --git a/Gemfile b/Gemfile index ba3c61d..195f03e 100644 --- a/Gemfile +++ b/Gemfile @@ -32,6 +32,10 @@ gem "oauth" # for parsing incoming mail gem "mail" +# for planet rss aggregation +gem "feed-normalizer" +gem "loofah" + group :test, :development do gem "rspec-rails", "~> 2.6" gem "machinist" diff --git a/Gemfile.lock b/Gemfile.lock index 372bafd..ccf5b51 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -39,7 +39,11 @@ GEM execjs (2.2.1) faker (1.4.2) i18n (~> 0.5) + feed-normalizer (1.5.2) + hpricot (>= 0.6) + simple-rss (>= 1.1) hike (1.2.3) + hpricot (0.8.6) htmlentities (4.3.2) i18n (0.6.11) innertube (1.1.0) @@ -48,6 +52,8 @@ GEM thor (>= 0.14, < 2.0) json (1.8.1) kgio (2.9.2) + loofah (2.0.0) + nokogiri (>= 1.5.9) machinist (2.0) mail (2.5.4) mime-types (~> 1.16) @@ -97,6 +103,7 @@ GEM rspec-core (~> 2.99.0) rspec-expectations (~> 2.99.0) rspec-mocks (~> 2.99.0) + simple-rss (1.3.1) sprockets (2.12.1) hike (~> 1.2) multi_json (~> 1.0) @@ -136,8 +143,10 @@ DEPENDENCIES dynamic_form exception_notification faker + feed-normalizer htmlentities jquery-rails + loofah machinist mail mysql2 (>= 0.3.14) diff --git a/app/assets/stylesheets/application.css b/app/assets/stylesheets/application.css index ce754cb..4130afe 100644 --- a/app/assets/stylesheets/application.css +++ b/app/assets/stylesheets/application.css @@ -494,6 +494,10 @@ li .domain { font-size: 8.5pt; vertical-align: middle; } +li .domain a { + color: #888; + text-decoration: none; +} .merge { background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAOCAYAAAD5YeaVAAAD8GlDQ1BJQ0MgUHJvZmlsZQAAOMuNVd1v21QUP4lvXKQWP6Cxjg4Vi69VU1u5GxqtxgZJk6XpQhq5zdgqpMl1bhpT1za2021Vn/YCbwz4A4CyBx6QeEIaDMT2su0BtElTQRXVJKQ9dNpAaJP2gqpwrq9Tu13GuJGvfznndz7v0TVAx1ea45hJGWDe8l01n5GPn5iWO1YhCc9BJ/RAp6Z7TrpcLgIuxoVH1sNfIcHeNwfa6/9zdVappwMknkJsVz19HvFpgJSpO64PIN5G+fAp30Hc8TziHS4miFhheJbjLMMzHB8POFPqKGKWi6TXtSriJcT9MzH5bAzzHIK1I08t6hq6zHpRdu2aYdJYuk9Q/881bzZa8Xrx6fLmJo/iu4/VXnfH1BB/rmu5ScQvI77m+BkmfxXxvcZcJY14L0DymZp7pML5yTcW61PvIN6JuGr4halQvmjNlCa4bXJ5zj6qhpxrujeKPYMXEd+q00KR5yNAlWZzrF+Ie+uNsdC/MO4tTOZafhbroyXuR3Df08bLiHsQf+ja6gTPWVimZl7l/oUrjl8OcxDWLbNU5D6JRL2gxkDu16fGuC054OMhclsyXTOOFEL+kmMGs4i5kfNuQ62EnBuam8tzP+Q+tSqhz9SuqpZlvR1EfBiOJTSgYMMM7jpYsAEyqJCHDL4dcFFTAwNMlFDUUpQYiadhDmXteeWAw3HEmA2s15k1RmnP4RHuhBybdBOF7MfnICmSQ2SYjIBM3iRvkcMki9IRcnDTthyLz2Ld2fTzPjTQK+Mdg8y5nkZfFO+se9LQr3/09xZr+5GcaSufeAfAww60mAPx+q8u/bAr8rFCLrx7s+vqEkw8qb+p26n11Aruq6m1iJH6PbWGv1VIY25mkNE8PkaQhxfLIF7DZXx80HD/A3l2jLclYs061xNpWCfoB6WHJTjbH0mV35Q/lRXlC+W8cndbl9t2SfhU+Fb4UfhO+F74GWThknBZ+Em4InwjXIyd1ePnY/Psg3pb1TJNu15TMKWMtFt6ScpKL0ivSMXIn9QtDUlj0h7U7N48t3i8eC0GnMC91dX2sTivgloDTgUVeEGHLTizbf5Da9JLhkhh29QOs1luMcScmBXTIIt7xRFxSBxnuJWfuAd1I7jntkyd/pgKaIwVr3MgmDo2q8x6IdB5QH162mcX7ajtnHGN2bov71OU1+U0fqqoXLD0wX5ZM005UHmySz3qLtDqILDvIL+iH6jB9y2x83ok898GOPQX3lk3Itl0A+BrD6D7tUjWh3fis58BXDigN9yF8M5PJH4B8Gr79/F/XRm8m241mw/wvur4BGDj42bzn+Vmc+NL9L8GcMn8F1kAcXi1s/XUAAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAAsSAAALEgHS3X78AAAAB3RJTUUH3gQIECAcSXeCTQAAAdhJREFUKM9dkDFoE1Ecxr/vvbvkBiEgFaSDg6uDBBJvEl2kOBaEIrgEAle4xxUKFnGR4mCCLiFZ3MVCBzOIdHBx6CIhgoNKQESsS7GKLVbv8u69v4NNof7Gb/h9fB9FBIPBIAHQwT/uGGMe4z9IkoPB4KKIbAM4BQACOSR4yRjzvtfrtUXkrFJqK8uyMfv9/pUwDF/leS4AWK1WUZbl1bIsL4dheF8phaIoPgFYUmEYvrPWvgyCgGEYoizL5yR3RMR471EUxZ8ois5TeF0lSbLnnLvpvf/ivf86nU5vzc/Pfya5SxIiEllrIZQd1e129crKyncA30jurq6uHiwuLnqSaVmWH0jCe/+oVqttqPF47GaLvfcAgCPBttb6dRAEh1EU3Wu1WoXa3NxEu90+vihJEgA4kc1Q3W5XO3csh7X2RMuM9fV1BJPJxDWbzdPOuTMAEMfx3Gg02ms0GnXn3BKAF8vLy79FBEEcx3PT6fSp1vrckflJvV5/5r2/S1IAPBARDIdDFVhrL1QqlWt5ngOAVCqVBRFZEJGPJG+kafqGJEXEK5I/rbW/tNbQWtM5t++cW1NKxcaYLZIEICShjDFvSd4GsA/ggORalmUP0zT90el0NACZjfwLrcfo3fIgR6gAAAAASUVORK5CYII=) no-repeat; @@ -1121,3 +1125,10 @@ div.fieldWithErrors { font-style: italic; padding-left: 0.5em; } + + + +div.weblog_content { + padding: 0 2em; + overflow-x: auto; +} diff --git a/app/controllers/settings_controller.rb b/app/controllers/settings_controller.rb index 945324f..7cbaed8 100644 --- a/app/controllers/settings_controller.rb +++ b/app/controllers/settings_controller.rb @@ -38,7 +38,7 @@ private :email_replies, :email_messages, :email_mentions, :pushover_replies, :pushover_messages, :pushover_mentions, :pushover_user_key, :pushover_device, :pushover_sound, - :mailing_list_mode, + :mailing_list_mode, :weblog_feed_url ) end end diff --git a/app/controllers/weblogs_controller.rb b/app/controllers/weblogs_controller.rb new file mode 100644 index 0000000..8345690 --- /dev/null +++ b/app/controllers/weblogs_controller.rb @@ -0,0 +1,24 @@ +class WeblogsController < ApplicationController + before_filter { @page = page } + + WEBLOGS_PER_PAGE = 10 + + def index + @pages = (Weblog.count / WEBLOGS_PER_PAGE.to_f).ceil + if @page > @pages + @page = @pages + end + @show_more = @page < @pages + + @weblogs = Weblog.order("created_at DESC"). + offset((@page - 1) * WEBLOGS_PER_PAGE). + limit(WEBLOGS_PER_PAGE) + + render :action => "index" + end + +private + def page + params[:page].to_i > 0 ? params[:page].to_i : 1 + end +end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 2a451b9..00ce8b8 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -16,4 +16,8 @@ module ApplicationHelper label_tag(nil, time_ago_in_words(*args), :title => args.first.strftime("%F %T %z")) end + + def main_root_url + Rails.application.routes.url_helpers.root_url + end end diff --git a/app/models/user.rb b/app/models/user.rb index f3abc17..ebbc906 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -19,6 +19,7 @@ class User < ActiveRecord::Base belongs_to :banned_by_user, :class_name => "User" has_many :invitations + has_many :weblogs has_secure_password @@ -31,6 +32,10 @@ class User < ActiveRecord::Base :format => { :with => /\A[A-Za-z0-9][A-Za-z0-9_-]{0,24}\Z/ }, :uniqueness => { :case_sensitive => false } + validates :weblog_feed_url, + :format => { :with => /\Ahttps?:\/\/([^\.]+\.)+[a-z]+(\/|\z)/ }, + :allow_blank => true + validates_each :username do |record,attr,value| if BANNED_USERNAMES.include?(value.to_s.downcase) record.errors.add(attr, "is not permitted") diff --git a/app/models/weblog.rb b/app/models/weblog.rb new file mode 100644 index 0000000..7bca796 --- /dev/null +++ b/app/models/weblog.rb @@ -0,0 +1,9 @@ +class Weblog < ActiveRecord::Base + belongs_to :user + + serialize :tags, Array + + def sanitized_content + Loofah.fragment(self.content).scrub!(:strip).scrub!(:nofollow).to_s.html_safe + end +end diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index d4e2dc0..adf048f 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -65,7 +65,7 @@ -
"> +
>Filters diff --git a/app/views/layouts/weblogs.html.erb b/app/views/layouts/weblogs.html.erb new file mode 100644 index 0000000..75da804 --- /dev/null +++ b/app/views/layouts/weblogs.html.erb @@ -0,0 +1,63 @@ + + + + + + + + + + + + + <% if @short_url %> + + <% end %> + <%= @title.present? ? "#{@title} | " : "" %>Planet <%= + Rails.application.name %> + <%= stylesheet_link_tag "application", :media => "all" %> + <% if @user %> + <%= javascript_include_tag "application" %> + + <% end %> + <%= csrf_meta_tags %> + + +
+ + +
+ <% if flash[:error] %> +
<%= flash[:error] %>
+ <% elsif flash[:success] %> +
<%= flash[:success] %>
+ <% end %> + + <%= yield %> + +
+
+
+ + + diff --git a/app/views/settings/index.html.erb b/app/views/settings/index.html.erb index 413c56d..b3b7578 100644 --- a/app/views/settings/index.html.erb +++ b/app/views/settings/index.html.erb @@ -38,11 +38,16 @@
+
+
<%= f.label :pushover_user_key, raw("Pushover User Key:"), :class => "required" %> <%= f.text_field :pushover_user_key, :size => 40 %> + + For optional comment and message notifications below +
@@ -58,6 +63,20 @@ <%= f.select :pushover_sound, Pushover.sounds.map{|k,v| [ v, k ] } %>
+
+ +
+ <%= f.label :weblog_feed_url, "Weblog Feed URL:", + :class => "required" %> + <%= f.text_field :weblog_feed_url, :placeholder => "optional", + :size => 40 %> + + For RSS/Atom feed aggregation; restrict to a technical feed if possible + +
+ +
+
<%= f.label :about, "About:", :class => "required" %> <%= f.text_area :about, :size => "100x5", :style => "width: 600px;" %> diff --git a/app/views/weblogs/index.html.erb b/app/views/weblogs/index.html.erb new file mode 100644 index 0000000..5a1968d --- /dev/null +++ b/app/views/weblogs/index.html.erb @@ -0,0 +1,44 @@ +
    +<% @weblogs.each do |weblog| %> +
  1. +
    +
    + + <%= weblog.title %> + + + <% weblog.tags.map{|t| t.downcase }.sort.each do |tag| %> + <%= tag %> + <% end %> + + + <%= weblog.site_title %> + +
    + by <%= + weblog.user.username %> + <%= raw(time_ago_in_words_label(weblog.created_at). + gsub(/^about /, "")) %> ago +
    +
    +
    +
    + <%= weblog.sanitized_content %> +
    +
    2. +<% end %> +
+ +
+ <% if @page && @page > 1 %> + "><< Page + <%= @page - 1 %> + <% end %> + + <% if @show_more %> + <% if @page && @page > 1 %> + | + <% end %> + Page <%= @page + 1 %> >> + <% end %> +
diff --git a/config/routes.rb b/config/routes.rb index d84d527..e9fa5bd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,111 +1,125 @@ Lobsters::Application.routes.draw do - scope :format => "html" do - root :to => "home#index", + constraints(:subdomain => "planet") do + root :to => "weblogs#index", + :subdomain => "planet", + :as => "weblogs_root", :protocol => (Rails.env == "production" ? "https://" : "http://") - get "/rss" => "home#index", :format => "rss" - get "/hottest" => "home#index", :format => "json" + get "/page/:page" => "weblogs#index" + end - get "/page/:page" => "home#index" + constraints do + scope :format => "html" do + root :to => "home#index", + :protocol => (Rails.env == "production" ? "https://" : "http://"), + :as => "root" - get "/newest" => "home#newest", :format => /html|json|rss/ - get "/newest/page/:page" => "home#newest" - get "/newest/:user" => "home#newest_by_user" - get "/newest/:user/page/:page" => "home#newest_by_user" - get "/recent" => "home#recent" - get "/recent/page/:page" => "home#recent" - get "/hidden" => "home#hidden" - get "/hidden/page/:page" => "home#hidden" + get "/rss" => "home#index", :format => "rss" + get "/hottest" => "home#index", :format => "json" - get "/top" => "home#top" - get "/top/page/:page" => "home#top" - get "/top/:length" => "home#top" - get "/top/:length/page/:page" => "home#top" + get "/page/:page" => "home#index" - get "/threads" => "comments#threads" - get "/threads/:user" => "comments#threads" + get "/newest" => "home#newest", :format => /html|json|rss/ + get "/newest/page/:page" => "home#newest" + get "/newest/:user" => "home#newest_by_user" + get "/newest/:user/page/:page" => "home#newest_by_user" + get "/recent" => "home#recent" + get "/recent/page/:page" => "home#recent" + get "/hidden" => "home#hidden" + get "/hidden/page/:page" => "home#hidden" - get "/login" => "login#index" - post "/login" => "login#login" - post "/logout" => "login#logout" + get "/top" => "home#top" + get "/top/page/:page" => "home#top" + get "/top/:length" => "home#top" + get "/top/:length/page/:page" => "home#top" - get "/signup" => "signup#index" - post "/signup" => "signup#signup" - get "/signup/invite" => "signup#invite" + get "/threads" => "comments#threads" + get "/threads/:user" => "comments#threads" - get "/login/forgot_password" => "login#forgot_password", - :as => "forgot_password" - post "/login/reset_password" => "login#reset_password", - :as => "reset_password" - match "/login/set_new_password" => "login#set_new_password", - :as => "set_new_password", :via => [:get, :post] + get "/login" => "login#index" + post "/login" => "login#login" + post "/logout" => "login#logout" - get "/t/:tag" => "home#tagged", :as => "tag", :format => /html|rss/ - get "/t/:tag/page/:page" => "home#tagged" + get "/signup" => "signup#index" + post "/signup" => "signup#signup" + get "/signup/invite" => "signup#invite" - get "/search" => "search#index" + get "/login/forgot_password" => "login#forgot_password", + :as => "forgot_password" + post "/login/reset_password" => "login#reset_password", + :as => "reset_password" + match "/login/set_new_password" => "login#set_new_password", + :as => "set_new_password", :via => [:get, :post] - resources :stories do - post "upvote" - post "downvote" - post "unvote" - post "undelete" - post "hide" - post "unhide" - end - post "/stories/fetch_url_title", :format => "json" - post "/stories/preview" => "stories#preview" + get "/t/:tag" => "home#tagged", :as => "tag", :format => /html|rss/ + get "/t/:tag/page/:page" => "home#tagged" - resources :comments do - member do - get "reply" + get "/search" => "search#index" + + resources :stories do post "upvote" post "downvote" post "unvote" - - post "delete" post "undelete" + post "hide" + post "unhide" end + post "/stories/fetch_url_title", :format => "json" + post "/stories/preview" => "stories#preview" + + resources :comments do + member do + get "reply" + post "upvote" + post "downvote" + post "unvote" + + post "delete" + post "undelete" + end + end + get "/comments/page/:page" => "comments#index" + + get "/messages/sent" => "messages#sent" + post "/messages/batch_delete" => "messages#batch_delete", + :as => "batch_delete_messages" + resources :messages do + post "keep_as_new" + end + + get "/s/:id/:title/comments/:comment_short_id" => "stories#show" + get "/s/:id/(:title)" => "stories#show", :format => /html|json/ + + get "/u" => "users#tree" + get "/u/:username" => "users#show", :as => "user" + + get "/settings" => "settings#index" + post "/settings" => "settings#update" + post "/settings/delete_account" => "settings#delete_account", + :as => "delete_account" + + get "/filters" => "filters#index" + post "/filters" => "filters#update" + + post "/invitations" => "invitations#create" + get "/invitations" => "invitations#index" + get "/invitations/request" => "invitations#build" + post "/invitations/create_by_request" => "invitations#create_by_request", + :as => "create_invitation_by_request" + get "/invitations/confirm/:code" => "invitations#confirm_email" + post "/invitations/send_for_request" => "invitations#send_for_request", + :as => "send_invitation_for_request" + get "/invitations/:invitation_code" => "signup#invited" + post "/invitations/delete_request" => "invitations#delete_request", + :as => "delete_invitation_request" + + get "/moderations" => "moderations#index" + get "/moderations/page/:page" => "moderations#index" + + get "/privacy" => "home#privacy" + get "/about" => "home#about" + + get "/resettraffic" => "home#resettraffic" end - get "/comments/page/:page" => "comments#index" - - get "/messages/sent" => "messages#sent" - post "/messages/batch_delete" => "messages#batch_delete", - :as => "batch_delete_messages" - resources :messages do - post "keep_as_new" - end - - get "/s/:id/:title/comments/:comment_short_id" => "stories#show" - get "/s/:id/(:title)" => "stories#show", :format => /html|json/ - - get "/u" => "users#tree" - get "/u/:username" => "users#show", :as => "user" - - get "/settings" => "settings#index" - post "/settings" => "settings#update" - post "/settings/delete_account" => "settings#delete_account", - :as => "delete_account" - - get "/filters" => "filters#index" - post "/filters" => "filters#update" - - post "/invitations" => "invitations#create" - get "/invitations" => "invitations#index" - get "/invitations/request" => "invitations#build" - post "/invitations/create_by_request" => "invitations#create_by_request", - :as => "create_invitation_by_request" - get "/invitations/confirm/:code" => "invitations#confirm_email" - post "/invitations/send_for_request" => "invitations#send_for_request", - :as => "send_invitation_for_request" - get "/invitations/:invitation_code" => "signup#invited" - post "/invitations/delete_request" => "invitations#delete_request", - :as => "delete_invitation_request" - - get "/moderations" => "moderations#index" - get "/moderations/page/:page" => "moderations#index" - - get "/privacy" => "home#privacy" - get "/about" => "home#about" end end diff --git a/db/migrate/20140701153554_add_user_weblog_url.rb b/db/migrate/20140701153554_add_user_weblog_url.rb new file mode 100644 index 0000000..f5899c2 --- /dev/null +++ b/db/migrate/20140701153554_add_user_weblog_url.rb @@ -0,0 +1,5 @@ +class AddUserWeblogUrl < ActiveRecord::Migration + def change + add_column :users, :weblog_feed_url, :string, :length => 500 + end +end diff --git a/db/migrate/20140804005415_add_weblogs.rb b/db/migrate/20140804005415_add_weblogs.rb new file mode 100644 index 0000000..28fff02 --- /dev/null +++ b/db/migrate/20140804005415_add_weblogs.rb @@ -0,0 +1,21 @@ +class AddWeblogs < ActiveRecord::Migration + def change + create_table :weblogs do |t| + t.timestamps + t.integer :user_id + t.string :title, :limit => 512 + t.string :url, :limit => 512 + t.string :site_title, :limit => 512 + t.string :site_url, :limit => 512 + t.text :content, :limit => 16777215 # mediumtext + t.text :tags + t.string :uuid + end + + # why can't the charset be specified in the create_table? + execute("ALTER TABLE weblogs MODIFY `uuid` varchar(200) CHARACTER SET utf8") + + add_index "weblogs", ["user_id", "uuid"], :name => "user_and_uuid", + :unique => true + end +end diff --git a/db/schema.rb b/db/schema.rb index 84f5ddc..37ddb2c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20140408160306) do +ActiveRecord::Schema.define(version: 20140804005415) do create_table "comments", force: true do |t| t.datetime "created_at", null: false @@ -167,6 +167,7 @@ ActiveRecord::Schema.define(version: 20140408160306) do t.string "banned_reason", limit: 200 t.datetime "deleted_at" t.string "pushover_sound" + t.string "weblog_feed_url" end add_index "users", ["mailing_list_mode"], name: "mailing_list_enabled", using: :btree @@ -187,4 +188,19 @@ ActiveRecord::Schema.define(version: 20140408160306) do add_index "votes", ["user_id", "comment_id"], name: "user_id_comment_id", using: :btree add_index "votes", ["user_id", "story_id"], name: "user_id_story_id", using: :btree + create_table "weblogs", force: true do |t| + t.datetime "created_at" + t.datetime "updated_at" + t.integer "user_id" + t.string "title", limit: 512 + t.string "url", limit: 512 + t.string "site_title", limit: 512 + t.string "site_url", limit: 512 + t.text "content", limit: 16777215 + t.text "tags" + t.string "uuid", limit: 200 + end + + add_index "weblogs", ["user_id", "uuid"], name: "user_and_uuid", unique: true, using: :btree + end diff --git a/script/update_weblogs b/script/update_weblogs new file mode 100755 index 0000000..b2e1cf8 --- /dev/null +++ b/script/update_weblogs @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby + +ENV["RAILS_ENV"] ||= "production" + +APP_PATH = File.expand_path('../../config/application', __FILE__) +require File.expand_path('../../config/boot', __FILE__) +require APP_PATH +Rails.application.require_environment! + +require "feed-normalizer" + +User.where("weblog_feed_url <> ''").each do |u| + begin + s = Sponge.new + data = s.fetch(u.weblog_feed_url, :get, nil, nil, { + "User-Agent" => "Mozilla/5.0 (compatible; Planet Lobsters; " << + "https://#{Rails.application.domain}/)" }) + if !data + raise "no data returned" + end + + feed = FeedNormalizer::FeedNormalizer.parse(data) + if !feed + raise "could not parse feed" + end + rescue => e + puts "error updating user #{u.id} from #{u.weblog_feed_url}: #{e}" + # TODO: send the user a private message on failure, probably limit to every + # x tries/days + next + end + + feed.entries.each do |e| + w = u.weblogs.where(:uuid => e.id).first + if !w + w = u.weblogs.build(:uuid => e.id) + end + + # prevent entries with bogus or future times from dominating the front page + if e.last_updated > Time.now + puts "skipping future entry #{e.id} with time #{e.last_updated}" + next + end + + # TODO: compare these manually, AR seems to always mark them dirty even if + # the content is the same + w.created_at = e.last_updated + w.url = e.url + w.title = e.title + w.content = e.content + w.tags = e.categories + + w.site_url = feed.url + w.site_title = feed.title + + w.save + end +end