use nokogiri to get doc titles

Gemfile

@@ -22,6 +22,8 @@ gem "unicorn"
 # for asset compilation
 gem "uglifier"
 
+gem "nokogiri"
+
 group :test, :development do
   gem "rspec-rails", "~> 2.6"
   gem "machinist"

Gemfile.lock

@@ -54,6 +54,7 @@ GEM
     mime-types (1.18)
     multi_json (1.1.0)
     mysql2 (0.3.11)
+    nokogiri (1.5.4)
     polyglot (0.3.3)
     rack (1.4.1)
     rack-cache (1.2)
@@ -123,6 +124,7 @@ DEPENDENCIES
   jquery-rails
   machinist
   mysql2
+  nokogiri
   rails (= 3.2.2)
   rspec-rails (~> 2.6)
   sqlite3

app/models/story.rb

@@ -15,7 +15,7 @@ class Story < ActiveRecord::Base
   # after this many minutes old, a story cannot be edited
   MAX_EDIT_MINS = 30
 
-	attr_accessor :vote, :story_type, :already_posted_story
+  attr_accessor :vote, :story_type, :already_posted_story, :fetched_content
   attr_accessor :tags_to_add, :tags_to_delete
 
   after_save :deal_with_tags
@@ -25,7 +25,7 @@ class Story < ActiveRecord::Base
     if self.url.present?
       # URI.parse is not very lenient, so we can't use it
 
-      if self.url.match(/\Ahttps?:\/\/[^\.]+\.[a-z]+\//)
+      if self.url.match(/\Ahttps?:\/\/([^\.]+\.)+[a-z]+(\/|\z)/)
         if (s = Story.find_by_url(self.url)) &&
         (Time.now - s.created_at) < 30.days
           errors.add(:url, "has already been submitted recently")
@@ -34,6 +34,8 @@ class Story < ActiveRecord::Base
       else
         errors.add(:url, "is not valid")
       end
+    elsif self.description.to_s.strip == ""
+      self.errors(:description, "must contain text if no URL posted")
     end
   end
 
@@ -91,8 +93,24 @@ class Story < ActiveRecord::Base
     end
   end
 
-  UP_RANGE = 400
-  DOWN_RANGE = 100
+  def fetched_title(for_remote_ip = nil)
+    doc = Nokogiri::HTML(fetched_content(for_remote_ip).to_s)
+    return doc.at_css("title").text  
+  end
+
+  def fetched_content(for_remote_ip = nil)
+    return @fetched_content if @fetched_content
+
+    begin
+      s = Sponge.new
+      s.timeout = 3
+      @fetched_content = s.fetch(self.url, :get, nil, nil,
+        { "User-agent" => "lobste.rs! for #{for_remote_ip}" }, 3)
+    rescue
+    end
+
+    @fetched_content
+  end
 
   def hotness
     score = upvotes - downvotes
@@ -116,6 +134,7 @@ class Story < ActiveRecord::Base
   def tags_a
     tags.map{|t| t.tag }
   end
+
   def tags_a=(new_tags)
     self.tags_to_delete = []
     self.tags_to_add = []
@@ -142,10 +161,9 @@ class Story < ActiveRecord::Base
 
   def title_as_url
     u = self.title.downcase.gsub(/[^a-z0-9_-]/, "_")
-		while self.title.match(/__/)
-			self.title.gsub!("__", "_")
+    while u.match(/__/)
+      u.gsub!("__", "_")
     end
-
     u
   end
 
@@ -173,8 +191,4 @@ class Story < ActiveRecord::Base
     Keystore.put("story:#{self.id}:comment_count",
       Comment.where(:story_id => self.id).count)
   end
-
-	def flag!
-    Story.update_counters self.id, :flaggings => 1
-  end
 end