From 2f190f9593535664fa83f9417054fff2f5190542 Mon Sep 17 00:00:00 2001
From: Russell Jarvis <rjjarvis@asu.edu>
Date: Sat, 27 Jun 2020 22:07:32 +1000
Subject: [PATCH] contents
---
science_access/get_bmark_corpus.py | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/science_access/get_bmark_corpus.py b/science_access/get_bmark_corpus.py
index f8e6933..b0fb4f2 100644
--- a/science_access/get_bmark_corpus.py
+++ b/science_access/get_bmark_corpus.py
@@ -11,16 +11,24 @@ from .crawl import collect_pubs, convert_pdf_to_txt#,process
from .scrape import get_driver
from .t_analysis import text_proc
from .utils import black_string
-
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+if 'DYNO' in os.environ:
+ heroku = False
+else:
+ heroku = True
def process(link):
urlDat = {}
urlDat['link'] = link
urlDat['page_rank'] = 'benchmark'
+ if heroku:
+ wait = WebDriverWait(driver, 10)
+ wait.until(lambda driver: driver.current_url != link)
+ link = driver.current_url
if str('pdf') not in link:
-
driver = get_driver()
-
driver.get(link)
+
crude_html = driver.page_source
soup = BeautifulSoup(crude_html, 'html.parser')
--
GitLab