Skip to content
Snippets Groups Projects
Commit 1819b964 authored by CI-bot's avatar CI-bot Committed by renku 0.10.4
Browse files

renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json

parent af81e477
No related branches found
No related tags found
1 merge request!238Automatic update - auto-update_2020-07-07_07-46
Pipeline #45275 passed with stage
Stage:
    in 40 seconds
    Hide whitespace changes
    Inline Side-by-side
    .renku/workflow/bb2335d7ac1a434593b661882db2586c.cwl 0 → 100644
    + 120
    − 0
    View file @ 1819b964
    class: Workflow
    cwlVersion: v1.0
    hints: []
    inputs:
    input_1:
    default: states-metadata.json
    streamable: false
    type: string
    input_2:
    default: out_folder
    streamable: false
    type: string
    input_3:
    default: data/covidtracking
    streamable: false
    type: string
    input_4:
    default:
    class: File
    path: ../../notebooks/process/download-covidtracking-data.ipynb
    streamable: false
    type: File
    input_5:
    default: runs/download-covidtracking-data.runs.ipynb
    streamable: false
    type: string
    input_6:
    default: states-daily.json
    streamable: false
    type: string
    outputs:
    output_0:
    outputSource: step_2/output_1
    streamable: false
    type: Directory
    output_1:
    outputSource: step_2/output_0
    streamable: false
    type: File
    requirements: []
    steps:
    step_1:
    in:
    filename: input_1
    input_directory: step_2/output_1
    out:
    - output_file
    run:
    arguments: []
    baseCommand:
    - 'true'
    class: CommandLineTool
    cwlVersion: v1.0
    hints: []
    inputs:
    filename:
    default: states-metadata.json
    streamable: false
    type: string
    input_directory:
    streamable: false
    type: Directory
    outputs:
    output_file:
    outputBinding:
    glob: $(inputs.filename)
    streamable: false
    type: File
    permanentFailCodes: []
    requirements:
    - &id001
    class: InlineJavascriptRequirement
    - &id002
    class: InitialWorkDirRequirement
    listing: $(inputs.input_directory.listing)
    successCodes: []
    temporaryFailCodes: []
    step_2:
    in:
    input_1: input_2
    input_2: input_3
    input_3: input_4
    input_4: input_5
    out:
    - output_0
    - output_1
    run: a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
    step_3:
    in:
    filename: input_6
    input_directory: step_2/output_1
    out:
    - output_file
    run:
    arguments: []
    baseCommand:
    - 'true'
    class: CommandLineTool
    cwlVersion: v1.0
    hints: []
    inputs:
    filename:
    default: states-daily.json
    streamable: false
    type: string
    input_directory:
    streamable: false
    type: Directory
    outputs:
    output_file:
    outputBinding:
    glob: $(inputs.filename)
    streamable: false
    type: File
    permanentFailCodes: []
    requirements:
    - *id001
    - *id002
    successCodes: []
    temporaryFailCodes: []
    data/covidtracking/states-daily.json LFS
    + 2
    − 2
    View file @ 1819b964
    source diff could not be displayed: it is stored in LFS. Options to address this: view the blob.
    data/covidtracking/states-metadata.json LFS
    + 2
    − 2
    View file @ 1819b964
    source diff could not be displayed: it is stored in LFS. Options to address this: view the blob.
    runs/download-covidtracking-data.runs.ipynb
    + 63
    − 63
    View file @ 1819b964
    %% Cell type:code id: tags:
    ``` python
    import requests
    import os
    import pandas as pd
    ```
    %% Cell type:code id: tags:parameters
    ``` python
    out_folder = "../data/covidtracking/"
    PAPERMILL_OUTPUT_PATH = None
    ```
    %% Cell type:code id: tags:injected-parameters
    ``` python
    # Parameters
    PAPERMILL_INPUT_PATH = "/tmp/h0x8je9l/notebooks/process/download-covidtracking-data.ipynb"
    PAPERMILL_INPUT_PATH = "/tmp/gm5bis_b/notebooks/process/download-covidtracking-data.ipynb"
    PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
    out_folder = "data/covidtracking"
    ```
    %% Cell type:markdown id: tags:
    # Download state metadata
    Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).
    %% Cell type:code id: tags:
    ``` python
    url = 'http://covidtracking.com/api/states/info'
    r = requests.get(url, allow_redirects=True)
    states_metadata_json = r.content
    ```
    %% Cell type:code id: tags:
    ``` python
    # save the result
    if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-metadata.json')
    with open(out_path, 'wb') as f:
    f.write(states_metadata_json)
    ```
    %% Cell type:code id: tags:
    ``` python
    metadata_df = pd.read_json(states_metadata_json)
    print(len(metadata_df), "states and territories have metadata")
    metadata_df.head(2)
    ```
    %% Output
    56 states and territories have metadata
    state notes \
    0 AK Total tests are taken from the annotations on ...
    0 AK Negatives = (Totals – Positives)\nPositives oc...
    1 AL Negatives = (Totals - Positives) \nPositives o...
    covid19Site \
    0 http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1 https://alpublichealth.maps.arcgis.com/apps/op...
    covid19SiteSecondary \
    0 http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1 https://dph1.adph.state.al.us/covid-19/
    covid19SiteTertiary twitter \
    0 https://alaska-dhss.maps.arcgis.com/apps/opsda... @Alaska_DHSS
    1 @alpublichealth
    covid19SiteOld name fips pui pum
    0 http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-... Alaska 2 False
    1 http://www.alabamapublichealth.gov/infectiousd... Alabama 1 False
    %% Cell type:markdown id: tags:
    # Download daily state data
    %% Cell type:code id: tags:
    ``` python
    url = 'https://covidtracking.com/api/states/daily'
    r = requests.get(url, allow_redirects=True)
    states_daily_json = r.content
    ```
    %% Cell type:code id: tags:
    ``` python
    # save the result
    if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-daily.json')
    with open(out_path, 'wb') as f:
    f.write(states_daily_json)
    ```
    %% Cell type:code id: tags:
    ``` python
    data_df = pd.read_json(states_daily_json)
    print(len(data_df), "data points")
    data_df.head(2)
    ```
    %% Output
    6849 data points
    6905 data points
    date state positive negative pending hospitalizedCurrently \
    0 20200705 AK 1111.0 121621.0 NaN 23.0
    1 20200705 AL 43953.0 405933.0 NaN 885.0
    0 20200706 AK 1138.0 122615.0 NaN 19.0
    1 20200706 AL 44878.0 410217.0 NaN 1016.0
    hospitalizedCumulative inIcuCurrently inIcuCumulative \
    0 NaN NaN NaN
    1 2909.0 NaN 843.0
    1 2914.0 NaN 843.0
    onVentilatorCurrently ... posNeg deathIncrease hospitalizedIncrease \
    0 3.0 ... 122732 0 0
    1 NaN ... 449886 0 3
    0 3.0 ... 123753 0 0
    1 NaN ... 455095 0 5
    hash commercialScore \
    0 60747b92beb9c600e7cb30085266b382691b2c3a 0
    1 261d349a43f69648cc120a21229dcccb34d1f9e2 0
    0 6337485c1dab1a0798672c8aa81f714d2b339df2 0
    1 d191311d4fceb2d234266af63c6554fdc93492e4 0
    negativeRegularScore negativeScore positiveScore score grade
    0 0 0 0 0
    1 0 0 0 0
    [2 rows x 39 columns]
    ......
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment

    This GitLab is a part of renkulab.io