From 7d1e17515ea02cc74ae42da1bc6efcf396eccae9 Mon Sep 17 00:00:00 2001 From: Clement Prevost Date: Tue, 13 Mar 2018 17:04:25 +0100 Subject: [PATCH 1/4] Allow ENV configuration of top directory --- newspaper/settings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/newspaper/settings.py b/newspaper/settings.py index 449d55b7..3b90e78c 100644 --- a/newspaper/settings.py +++ b/newspaper/settings.py @@ -32,7 +32,13 @@ DATA_DIRECTORY = '.newspaper_scraper' -TOP_DIRECTORY = os.path.join(tempfile.gettempdir(), DATA_DIRECTORY) + +if 'NEWSPAPER_SCRAPER_BASE_DIRECTORY' in os.environ: + BASE_DIRECTORY = os.environ['NEWSPAPER_SCRAPER_BASE_DIRECTORY'] +else: + BASE_DIRECTORY = tempfile.gettempdir() + +TOP_DIRECTORY = os.path.join(BASE_DIRECTORY, DATA_DIRECTORY) if not os.path.exists(TOP_DIRECTORY): os.mkdir(TOP_DIRECTORY) From d92b92d414845229bad1ca86db5da0690b25e0a1 Mon Sep 17 00:00:00 2001 From: Clement Prevost Date: Tue, 13 Mar 2018 17:17:44 +0100 Subject: [PATCH 2/4] Use writeable directory in lambda env --- newspaper/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/settings.py b/newspaper/settings.py index 3b90e78c..b02d86a0 100644 --- a/newspaper/settings.py +++ b/newspaper/settings.py @@ -33,8 +33,8 @@ DATA_DIRECTORY = '.newspaper_scraper' -if 'NEWSPAPER_SCRAPER_BASE_DIRECTORY' in os.environ: - BASE_DIRECTORY = os.environ['NEWSPAPER_SCRAPER_BASE_DIRECTORY'] +if 'LAMBDA_TASK_ROOT' in os.environ: + BASE_DIRECTORY = os.environ['LAMBDA_TASK_ROOT'] else: BASE_DIRECTORY = tempfile.gettempdir() From 407eb9cbda225676976ee6edb0f1fe9cd5bdcf8d Mon Sep 17 00:00:00 2001 From: Clement Prevost Date: Tue, 13 Mar 2018 17:46:27 +0100 Subject: [PATCH 3/4] Use writeable directory in lambda env --- newspaper/settings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/newspaper/settings.py b/newspaper/settings.py index b02d86a0..42cf98dd 100644 --- a/newspaper/settings.py +++ b/newspaper/settings.py @@ -34,7 +34,9 @@ if 'LAMBDA_TASK_ROOT' in os.environ: - BASE_DIRECTORY = os.environ['LAMBDA_TASK_ROOT'] + # https://docs.aws.amazon.com/lambda/latest/dg/limits.html + # only /tmp is writeable in lambda + BASE_DIRECTORY = "/tmp" else: BASE_DIRECTORY = tempfile.gettempdir() From 79f5f4ac71961aaed7b867372c33a1069ca7b52c Mon Sep 17 00:00:00 2001 From: Clement Prevost Date: Thu, 15 Mar 2018 11:48:50 +0100 Subject: [PATCH 4/4] Allow base dir config --- newspaper/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/settings.py b/newspaper/settings.py index 42cf98dd..e03d3fc5 100644 --- a/newspaper/settings.py +++ b/newspaper/settings.py @@ -33,10 +33,10 @@ DATA_DIRECTORY = '.newspaper_scraper' -if 'LAMBDA_TASK_ROOT' in os.environ: +if 'NEWSPAPER_BASE_DIRECTORY' in os.environ: # https://docs.aws.amazon.com/lambda/latest/dg/limits.html # only /tmp is writeable in lambda - BASE_DIRECTORY = "/tmp" + BASE_DIRECTORY = os.environ['NEWSPAPER_BASE_DIRECTORY'] else: BASE_DIRECTORY = tempfile.gettempdir()