From fd41bfa566e83adef3a6f9ef8407de1799b29e48 Mon Sep 17 00:00:00 2001 From: joshua stein Date: Tue, 25 Jun 2013 13:58:52 -0500 Subject: [PATCH] start on mailing list interface --- Gemfile | 8 +- Gemfile.lock | 1 + app/assets/stylesheets/80s.css | 158 +++++++++++++++++++++++++++++++ app/models/comment.rb | 12 ++- app/models/story.rb | 4 + extras/utils.rb | 13 +++ script/mail_new_activity | 165 +++++++++++++++++++++++++++++++++ script/parse_inbound_mail | 149 +++++++++++++++++++++++++++++ 8 files changed, 501 insertions(+), 9 deletions(-) create mode 100644 app/assets/stylesheets/80s.css create mode 100755 script/mail_new_activity create mode 100755 script/parse_inbound_mail diff --git a/Gemfile b/Gemfile index 7649475..4c15396 100644 --- a/Gemfile +++ b/Gemfile @@ -9,17 +9,14 @@ gem "rake", "10.0.3" gem "mysql2", :git => "git://github.com/brianmario/mysql2.git" -gem "jquery-rails" - -# To use ActiveModel has_secure_password gem "bcrypt-ruby", "3.0.0" +gem "jquery-rails" gem "dynamic_form" # use old version that doesn't have tinder bullshit gem "exception_notification", "2.6.1" -# Use unicorn as the app server gem "unicorn" # for asset compilation @@ -27,11 +24,12 @@ gem "uglifier" gem "nokogiri" gem "htmlentities" - gem "rdiscount" gem "thinking-sphinx", "2.0.12" +gem "mail" + group :test, :development do gem "rspec-rails", "~> 2.6" gem "machinist" diff --git a/Gemfile.lock b/Gemfile.lock index 81935c6..5e9d901 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -135,6 +135,7 @@ DEPENDENCIES htmlentities jquery-rails machinist + mail mysql2! nokogiri rails (= 3.2.13) diff --git a/app/assets/stylesheets/80s.css b/app/assets/stylesheets/80s.css new file mode 100644 index 0000000..05b6ecd --- /dev/null +++ b/app/assets/stylesheets/80s.css @@ -0,0 +1,158 @@ +div#wrapper { + background-color: #f6f6ef; + margin-bottom: 5px; +} + +body, textarea, input, button { + font-family: verdana; + color: #828282; +} +textarea, input, button { + font-family: monospace; + color: black; +} +input[type="submit"], button { + padding: 2px 5px; + text-transform: lowercase; +} +a { + color: black; +} +span.headerlinks { + font-weight: normal; + text-transform: lowercase; +} +span.headerlinks a:after { + content: " |"; +} +span.headerlinks a[href="/logout"]:after, +span.headerlinks a[href="/search"]:after { + content: ""; +} +span.headerlinks a { + padding-right: 3px !important; + padding-left: 0px !important; +} +div#header { + background-color: #ff6600; + padding: 3px 4px 2px 2px; +} +div#header a { + color: black; + font-size: 10pt; +} +a#l_holder { + background-color: transparent !important; + border: 1px solid white; + margin-top: -1px; +} +div#header a.cur_url { + color: white; +} +div#header span.headerlinks a[href="/"] { + font-weight: bold; + color: black !important; + font-size: 0px; +} +div#header span.headerlinks a[href="/"]:after { + content: "Lobsters"; + text-transform: none; + font-size: 10pt; + padding-right: 10px; +} + +ol.stories { + list-style: decimal; + padding-left: 32px; + font-size: 10pt; +} + +li.story { + padding-bottom: 0px; + margin-top: -3px; +} + +li.story div.details { + margin-left: 8px; +} + +li.story .link a { + font-size: 10pt; + font-weight: normal; +} +ol.stories.list li.story div.details span.link a:visited { + color: #828282; +} +li.story .byline { + font-size: 7.5pt; + padding-left: 0px; +} + +span.domain { + font-size: 8pt; + font-style: normal; +} +span.domain:before { + content: "("; +} +span.domain:after { + content: ")"; +} +span.domain:empty:before { + content: ""; +} +span.domain:empty:after { + content: ""; +} + +li.story div.voters { + width: 7px; + margin-left: -5px; +} + +li.story div.voters div.score { + position: absolute; + margin-left: 13px; + margin-top: 5px; + font-size: 7.5pt; + margin-bottom: 0; + color: #888; +} +li.story div.voters div.score:after { + content: " points"; +} +li.story div.voters a.upvoter { + border-width: 4px; + border-bottom-width: 8px; + display: block; + margin-left: 0px; + margin-top: 7px; + padding: 0; +} +li.story div.voters a.upvoter { + border-color: transparent transparent #999 transparent; +} +li.story div.voters a.upvoter:hover { + border-color: transparent transparent #999 transparent; +} +li.story.upvoted div.voters a.upvoter { + border-bottom-color: transparent; +} + + +li.story div.voters a.downvoter { + display: none; +} + +div.comment_text { + color: black; + font-size: 9pt; +} + +a.tag { + font-size: 7pt; +} + +.box .legend { + background-color: transparent; +} diff --git a/app/models/comment.rb b/app/models/comment.rb index 13703c7..1ac4947 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -280,15 +280,19 @@ class Comment < ActiveRecord::Base self.markeddown_comment = self.generated_markeddown_comment end + def has_been_edited? + self.updated_at && (self.updated_at - self.created_at > 1.minute) + end + + def mailing_list_message_id + "comment.#{short_id}.#{created_at.to_i}@lobste.rs" + end + def plaintext_comment # TODO: linkify then strip tags and convert entities back comment end - def has_been_edited? - self.updated_at && (self.updated_at - self.created_at > 1.minute) - end - def self.ordered_for_story_or_thread_for_user(story_id, thread_id, user) parents = {} diff --git a/app/models/story.rb b/app/models/story.rb index 8f45e79..3cdeaf2 100644 --- a/app/models/story.rb +++ b/app/models/story.rb @@ -283,6 +283,10 @@ class Story < ActiveRecord::Base self.markeddown_description = self.generated_markeddown_description end + def mailing_list_message_id + "story.#{short_id}.#{created_at.to_i}@lobste.rs" + end + @_tags_a = [] def tags_a @_tags_a ||= self.taggings.map{|t| t.tag.tag } diff --git a/extras/utils.rb b/extras/utils.rb index feb7fb4..de097dd 100644 --- a/extras/utils.rb +++ b/extras/utils.rb @@ -13,4 +13,17 @@ class Utils return str end + + def silence_stream(*streams) + on_hold = streams.collect {|stream| stream.dup } + streams.each do |stream| + stream.reopen("/dev/null") + stream.sync = true + end + yield + ensure + streams.each_with_index do |stream, i| + stream.reopen(on_hold[i]) + end + end end diff --git a/script/mail_new_activity b/script/mail_new_activity new file mode 100755 index 0000000..ef573bd --- /dev/null +++ b/script/mail_new_activity @@ -0,0 +1,165 @@ +#!/usr/bin/env ruby + +ENV["RAILS_ENV"] ||= "production" + +APP_PATH = File.expand_path('../../config/application', __FILE__) +require File.expand_path('../../config/boot', __FILE__) +require APP_PATH +Rails.application.require_environment! + +class String + def force_to_ascii + encode("us-ascii", :invalid => :replace, :undef => :replace, + :replace => "?") + end + + # like ActionView::Helpers::TextHelper but preserve > and indentation when + # wrapping lines + def word_wrap(len) + split("\n").collect do |line| + if line.length <= len + line + elsif m = line.match(/^(> ?| +)(.*)/) + ind = m[1] + if len - ind.length < 0 + ind = " " + end + m[2].gsub(/(.{1,#{len - ind.length}})(\s+|$)/, "#{ind}\\1\n").strip + else + line.gsub(/(.{1,#{len}})(\s+|$)/, "\\1\n").strip + end + end * "\n" + end +end + +EMAIL_WIDTH = 72 +LAST_STORY_KEY = "mailing:last_story_id" +LAST_COMMENT_KEY = "mailing:last_comment_id" + +mailing_list_users = User.where(:mailing_list_enabled => true) + +last_story_id = (Keystore.value_for(LAST_STORY_KEY) || Story.last.id).to_i + +Story.where("id > ?", last_story_id).order(:id).each do |s| + s.fetch_story_cache! + s.save + + mailing_list_users.each do |u| + if (s.tags.map{|t| t.id } & u.tag_filters.map{|t| t.tag_id }).any? + next + end + + IO.popen([ {}, "/usr/sbin/sendmail", "-i", "-f", "nobody@lobste.rs", + u.email ], "w") do |mail| + mail.puts "From: #{s.user.username} <#{s.user.username}@lobste.rs>" + mail.puts "Reply-To: lobsters-#{u.mailing_list_token}@lobste.rs" + mail.puts "To: lobsters-#{u.mailing_list_token}@lobste.rs" + mail.puts "X-BeenThere: lobsters-#{u.mailing_list_token}.lobste.rs" + mail.puts "List-Id: Lobsters " + mail.puts "List-Unsubscribe: " + mail.puts "Precedence: list" + mail.puts "Content-Type: text/plain; charset=\"us-ascii\"" + mail.puts "Message-ID: <#{s.mailing_list_message_id}>" + mail.puts "Date: " << s.created_at.strftime("%a, %d %b %Y %H:%M:%S %z") + mail.puts "Subject: #{s.title.force_to_ascii}" << + s.tags.sort_by{|t| t.tag }.map{|t| " [#{t.tag}]" }.join + + mail.puts "" + + if s.description.present? + mail.puts s.description.to_s.force_to_ascii.word_wrap(EMAIL_WIDTH) + end + + if s.url.present? + if s.description.present? + mail.puts "" + end + + mail.puts "Via: #{s.url}" + + if s.story_cache.present? + mail.puts "" + mail.puts s.story_cache.to_s.force_to_ascii.word_wrap(EMAIL_WIDTH) + end + end + + mail.puts "" + mail.puts "-- " + mail.puts "Vote: #{s.short_id_url}" + end + end + + last_story_id = s.id +end + +Keystore.put(LAST_STORY_KEY, last_story_id) + +# repeat for comments + +last_comment_id = (Keystore.value_for(LAST_COMMENT_KEY) || + Comment.last.id).to_i + +Comment.where("id > ?", last_comment_id).order(:id).each do |c| + mailing_list_users.each do |u| + if (c.story.tags.map{|t| t.id } & u.tag_filters.map{|t| t.tag_id }).any? + next + end + + IO.popen([ {}, "/usr/sbin/sendmail", "-i", "-f", "nobody@lobste.rs", + u.email ], "w") do |mail| + mail.puts "From: #{c.user.username} <#{c.user.username}@lobste.rs>" + mail.puts "Reply-To: lobsters-#{u.mailing_list_token}@lobste.rs" + mail.puts "To: lobsters-#{u.mailing_list_token}@lobste.rs" + mail.puts "List-Id: Lobsters " + mail.puts "List-Unsubscribe: " + mail.puts "Precedence: list" + mail.puts "Content-Type: text/plain; charset=\"us-ascii\"" + mail.puts "Message-ID: <#{c.mailing_list_message_id}>" + + refs = [ "<#{c.story.mailing_list_message_id}>" ] + + if c.parent_comment_id + mail.puts "In-Reply-To: <#{c.parent_comment.mailing_list_message_id}>" + + thread = [] + indent_level = 0 + Comment.ordered_for_story_or_thread_for_user(nil, c.thread_id, + nil).reverse.each do |cc| + if indent_level > 0 && cc.indent_level < indent_level + thread.unshift cc + indent_level = cc.indent_level + elsif cc.id == c.id + indent_level = cc.indent_level + end + end + + thread.each do |cc| + refs.push "<#{cc.mailing_list_message_id}>" + end + else + mail.puts "In-Reply-To: <#{c.story.mailing_list_message_id}>" + end + + mail.print "References:" + refs.each do |ref| + mail.puts " #{ref}" + end + + mail.puts "Date: " << c.created_at.strftime("%a, %d %b %Y %H:%M:%S %z") + mail.puts "Subject: Re: #{c.story.title.force_to_ascii}" << + c.story.tags.sort_by{|t| t.tag }.map{|t| " [#{t.tag}]" }.join + + mail.puts "" + + mail.puts c.comment.to_s.force_to_ascii.word_wrap(EMAIL_WIDTH) + + mail.puts "" + mail.puts "-- " + mail.puts "Vote: #{c.short_id_url}" + end + end + + last_comment_id = c.id +end + +Keystore.put(LAST_COMMENT_KEY, last_comment_id) diff --git a/script/parse_inbound_mail b/script/parse_inbound_mail new file mode 100755 index 0000000..e3a535f --- /dev/null +++ b/script/parse_inbound_mail @@ -0,0 +1,149 @@ +#!/usr/bin/env ruby +# +# postfix main.cf: +# relay_domains = lobste.rs +# transport_maps = hash:/etc/postfix/transport +# defer_transports = +# +# postfix transports: +# lobste.rs lobsters: +# +# postfix master.cf: +# lobsters unix - n n - 2 pipe +# flags=Fqhu user=lobsters size=1024000 argv=/d/rails/lobsters/script/parse_inbound_mail ${recipient} ${sender} + +ENV["RAILS_ENV"] ||= "production" + +APP_PATH = File.expand_path('../../config/application', __FILE__) +require File.expand_path('../../config/boot', __FILE__) +require APP_PATH +Rails.application.require_environment! + +# postfix exit codes +EX_NOUSER = 67 +EX_TEMPFAIL = 75 +EX_UNAVAILABLE = 69 + +recipient = ARGV[0] +user_token = recipient.gsub(/^lobsters-/, "").gsub(/@.*/, "") +sender = ARGV[1] +message = "" +email = nil + +while !STDIN.eof? + message += STDIN.gets.to_s +end + +if message.match(/^X-BeenThere: lobsters-/i) + # avoid looping + exit +end + +sending_user = User.where(:mailing_list_enabled => true, + :mailing_list_token => user_token).first + +if !sending_user + STDERR.puts "no user with mailing list token #{user_token}" + + # if this looks like a user token but invalid, generate a bounce to be + # helpful. otherwise supress it to avoid talking back to spammers + exit(recipient.match(/^lobsters-/) ? EX_NOUSER : 0) +end + +# the mail gem stupidly spams STDERR while parsing e-mail, so silence that +# stream to avoid anything getting back to postfix +begin + Utils.silence_stream(STDERR) do + email = Mail.read_from_string(message) + end + + if !email + raise + end +rescue + STDERR.puts "error parsing e-mail" + exit EX_UNAVAILABLE +end + +# figure out what this reply is to +irt = email[:in_reply_to].to_s.gsub(/[^A-Za-z0-9@\.]/, "") + +if m = irt.match(/^comment\.([^\.]+)\.\d+@/) + parent = Comment.find_by_short_id(m[1]) +elsif m = irt.match(/^story\.([^\.]+)\.\d+@/) + parent = Story.find_by_short_id(m[1]) +end + +if !parent + STDERR.puts "no valid comment or story being replied to" + exit EX_NOUSER +end + +body = nil +possible_charset = nil + +if email.multipart? + # parts[0] - multipart/alternative + # parts[0].parts[0] - text/plain + # parts[0].parts[1] - text/html + if (p = email.parts.first.parts.select{|p| + p.content_type.match(/text\/plain/) }).any? + begin + possible_charset = p.first.content_type_parameters["charset"] + rescue + end + + # parts[0] - text/plain + elsif (p = email.parts.select{|p| + p.content_type.match(/text\/plain/) }).any? + body = p.first.body.to_s + + begin + possible_charset = p.first.content_type_parameters["charset"] + rescue + end + end +elsif email.content_type.to_s.match(/text\/plain/) + body = email.body.to_s + + begin + possible_charset = email.content_type_parameters["charset"] + rescue + end + +elsif !email.content_type.to_s.present? + # no content-type header, assume it's text/plain + body = email.body.to_s +end + +if !body.present? + # oh well + STDERR.puts "no valid text/plain body found" + exit EX_UNAVAILABLE +end + +# try to remove sig lines +body.gsub!(/^-- \n.+\z/, "") + +# TODO: try to strip out attribution line, followed by an optional blank line, +# and then lines prefixed with > + +body.strip! + +c = Comment.new +c.user_id = sending_user.id +c.comment = body + +if parent.is_a?(Comment) + c.story_id = parent.story_id + c.parent_comment_id = parent.id +else + c.story_id = parent.id +end + +if c.save + exit +else + STDERR.puts c.errors.inspect + exit EX_UNAVAILABLE +end