use nokogiri to get doc titles
This commit is contained in:
parent
264cd85ece
commit
f801932a03
2
Gemfile
2
Gemfile
|
@ -22,6 +22,8 @@ gem "unicorn"
|
||||||
# for asset compilation
|
# for asset compilation
|
||||||
gem "uglifier"
|
gem "uglifier"
|
||||||
|
|
||||||
|
gem "nokogiri"
|
||||||
|
|
||||||
group :test, :development do
|
group :test, :development do
|
||||||
gem "rspec-rails", "~> 2.6"
|
gem "rspec-rails", "~> 2.6"
|
||||||
gem "machinist"
|
gem "machinist"
|
||||||
|
|
|
@ -54,6 +54,7 @@ GEM
|
||||||
mime-types (1.18)
|
mime-types (1.18)
|
||||||
multi_json (1.1.0)
|
multi_json (1.1.0)
|
||||||
mysql2 (0.3.11)
|
mysql2 (0.3.11)
|
||||||
|
nokogiri (1.5.4)
|
||||||
polyglot (0.3.3)
|
polyglot (0.3.3)
|
||||||
rack (1.4.1)
|
rack (1.4.1)
|
||||||
rack-cache (1.2)
|
rack-cache (1.2)
|
||||||
|
@ -123,6 +124,7 @@ DEPENDENCIES
|
||||||
jquery-rails
|
jquery-rails
|
||||||
machinist
|
machinist
|
||||||
mysql2
|
mysql2
|
||||||
|
nokogiri
|
||||||
rails (= 3.2.2)
|
rails (= 3.2.2)
|
||||||
rspec-rails (~> 2.6)
|
rspec-rails (~> 2.6)
|
||||||
sqlite3
|
sqlite3
|
||||||
|
|
|
@ -15,7 +15,7 @@ class Story < ActiveRecord::Base
|
||||||
# after this many minutes old, a story cannot be edited
|
# after this many minutes old, a story cannot be edited
|
||||||
MAX_EDIT_MINS = 30
|
MAX_EDIT_MINS = 30
|
||||||
|
|
||||||
attr_accessor :vote, :story_type, :already_posted_story
|
attr_accessor :vote, :story_type, :already_posted_story, :fetched_content
|
||||||
attr_accessor :tags_to_add, :tags_to_delete
|
attr_accessor :tags_to_add, :tags_to_delete
|
||||||
|
|
||||||
after_save :deal_with_tags
|
after_save :deal_with_tags
|
||||||
|
@ -25,7 +25,7 @@ class Story < ActiveRecord::Base
|
||||||
if self.url.present?
|
if self.url.present?
|
||||||
# URI.parse is not very lenient, so we can't use it
|
# URI.parse is not very lenient, so we can't use it
|
||||||
|
|
||||||
if self.url.match(/\Ahttps?:\/\/[^\.]+\.[a-z]+\//)
|
if self.url.match(/\Ahttps?:\/\/([^\.]+\.)+[a-z]+(\/|\z)/)
|
||||||
if (s = Story.find_by_url(self.url)) &&
|
if (s = Story.find_by_url(self.url)) &&
|
||||||
(Time.now - s.created_at) < 30.days
|
(Time.now - s.created_at) < 30.days
|
||||||
errors.add(:url, "has already been submitted recently")
|
errors.add(:url, "has already been submitted recently")
|
||||||
|
@ -34,6 +34,8 @@ class Story < ActiveRecord::Base
|
||||||
else
|
else
|
||||||
errors.add(:url, "is not valid")
|
errors.add(:url, "is not valid")
|
||||||
end
|
end
|
||||||
|
elsif self.description.to_s.strip == ""
|
||||||
|
self.errors(:description, "must contain text if no URL posted")
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -91,8 +93,24 @@ class Story < ActiveRecord::Base
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
UP_RANGE = 400
|
def fetched_title(for_remote_ip = nil)
|
||||||
DOWN_RANGE = 100
|
doc = Nokogiri::HTML(fetched_content(for_remote_ip).to_s)
|
||||||
|
return doc.at_css("title").text
|
||||||
|
end
|
||||||
|
|
||||||
|
def fetched_content(for_remote_ip = nil)
|
||||||
|
return @fetched_content if @fetched_content
|
||||||
|
|
||||||
|
begin
|
||||||
|
s = Sponge.new
|
||||||
|
s.timeout = 3
|
||||||
|
@fetched_content = s.fetch(self.url, :get, nil, nil,
|
||||||
|
{ "User-agent" => "lobste.rs! for #{for_remote_ip}" }, 3)
|
||||||
|
rescue
|
||||||
|
end
|
||||||
|
|
||||||
|
@fetched_content
|
||||||
|
end
|
||||||
|
|
||||||
def hotness
|
def hotness
|
||||||
score = upvotes - downvotes
|
score = upvotes - downvotes
|
||||||
|
@ -116,6 +134,7 @@ class Story < ActiveRecord::Base
|
||||||
def tags_a
|
def tags_a
|
||||||
tags.map{|t| t.tag }
|
tags.map{|t| t.tag }
|
||||||
end
|
end
|
||||||
|
|
||||||
def tags_a=(new_tags)
|
def tags_a=(new_tags)
|
||||||
self.tags_to_delete = []
|
self.tags_to_delete = []
|
||||||
self.tags_to_add = []
|
self.tags_to_add = []
|
||||||
|
@ -142,10 +161,9 @@ class Story < ActiveRecord::Base
|
||||||
|
|
||||||
def title_as_url
|
def title_as_url
|
||||||
u = self.title.downcase.gsub(/[^a-z0-9_-]/, "_")
|
u = self.title.downcase.gsub(/[^a-z0-9_-]/, "_")
|
||||||
while self.title.match(/__/)
|
while u.match(/__/)
|
||||||
self.title.gsub!("__", "_")
|
u.gsub!("__", "_")
|
||||||
end
|
end
|
||||||
|
|
||||||
u
|
u
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -173,8 +191,4 @@ class Story < ActiveRecord::Base
|
||||||
Keystore.put("story:#{self.id}:comment_count",
|
Keystore.put("story:#{self.id}:comment_count",
|
||||||
Comment.where(:story_id => self.id).count)
|
Comment.where(:story_id => self.id).count)
|
||||||
end
|
end
|
||||||
|
|
||||||
def flag!
|
|
||||||
Story.update_counters self.id, :flaggings => 1
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in a new issue