fix fetched_attributes fallout

This commit is contained in:
joshua stein 2015-12-03 14:54:55 -06:00
parent 8b4dec5746
commit 9f9e608fc8
3 changed files with 14 additions and 15 deletions

View file

@ -62,10 +62,7 @@ class StoriesController < ApplicationController
s.fetching_ip = request.remote_ip s.fetching_ip = request.remote_ip
s.url = params[:fetch_url] s.url = params[:fetch_url]
sattrs = s.fetched_attributes return render :json => s.fetched_attributes
sattrs.delete(:content)
return render :json => sattrs
end end
def new def new

View file

@ -727,17 +727,19 @@ class Story < ActiveRecord::Base
:title => "", :title => "",
} }
begin if !@fetched_content
s = Sponge.new begin
s.timeout = 3 s = Sponge.new
@fetched_attributes[:content] = s.fetch(self.url, :get, nil, nil, { s.timeout = 3
"User-agent" => "#{Rails.application.domain} for #{self.fetching_ip}" @fetched_content = s.fetch(self.url, :get, nil, nil, {
}, 3) "User-agent" => "#{Rails.application.domain} for #{self.fetching_ip}"
rescue }, 3)
return @fetched_attributes rescue
return @fetched_attributes
end
end end
parsed = Nokogiri::HTML(@fetched_attributes[:content].to_s) parsed = Nokogiri::HTML(@fetched_content.to_s)
# parse best title from html tags # parse best title from html tags
# try <meta property="og:title"> first, it probably won't have the site # try <meta property="og:title"> first, it probably won't have the site

View file

@ -93,12 +93,12 @@ describe Story do
s = Story.make s = Story.make
s.fetched_content = File.read(Rails.root + s.fetched_content = File.read(Rails.root +
"spec/fixtures/story_pages/1.html") "spec/fixtures/story_pages/1.html")
s.fetched_title.should == "B2G demo & quick hack // by Paul Rouget" s.fetched_attributes[:title].should == "B2G demo & quick hack // by Paul Rouget"
s = Story.make s = Story.make
s.fetched_content = File.read(Rails.root + s.fetched_content = File.read(Rails.root +
"spec/fixtures/story_pages/2.html") "spec/fixtures/story_pages/2.html")
s.fetched_title.should == "Google" s.fetched_attributes[:title].should == "Google"
end end
it "has an underscored url from its title" do it "has an underscored url from its title" do