.gitignore

-
 # Created by https://www.gitignore.io/api/macos,python,R,linux,vim,emacs
 
+src/python/test_debug.py
+data/AB_other/SessionOverviews_tar/*/previews
+
 ### Emacs ###
 # -*- mode: gitignore; -*-
 *~
@@ -278,3 +280,15 @@ tags
 # Pipfiles
 Pipfile
 Pipfile.lock
+
+
+# NER training files
+data/train_NER/1[0-9][0-9][0-9]_20[0-9][0-9][0-9][0-9][0-9][0-9].txt
+data/train_NER/1[0-9][0-9][0-9]_20[0-9][0-9][0-9][0-9][0-9][0-9]_french.txt
+
+
+# notunique files
+data/lists/notunique_*.txt
+
+# IDEs
+.idea/

.gitlab-ci.yml

@@ -24,8 +24,9 @@ image_build:
 
 dot:
   stage: build
-  image: renku/renku-python:0.2.0
+  image: renku/renku-python:latest
   script:
+    - renku --version
     - renku log --format dot $(git ls-files --no-empty-directory --recurse-submodules) > graph.dot
   artifacts:
     paths:

data/politicians/metadata.yml → .renku/datasets/25c2b603477d43eca051f0e8f44a2400/metadata.yml

@@ -25,7 +25,17 @@ authors:
   name: Lilian Gasser
 created: 2018-12-05 13:38:43.659847
 files:
-  Ratsmitglieder_1848_DE_corr.xlsx:
+  ../../../data/politicians/MPs_additionalInfo.csv:
+    '@type': http://schema.org/DigitalDocument
+    added: 2019-01-23 09:49:23.596009
+    authors:
+    - '@type': dcterms:creator
+      affiliation: null
+      email: gasserli@ethz.ch
+      name: Lilian Gasser
+    path: ../../../data/politicians/MPs_additionalInfo.csv
+    url: /home/lili/Desktop/MPs_additionalInfo.csv
+  ../../../data/politicians/Ratsmitglieder_1848_DE_corr.xlsx:
     '@type': http://schema.org/DigitalDocument
     added: 2018-12-05 13:41:01.027986
     authors:
@@ -33,8 +43,7 @@ files:
       affiliation: null
       email: gasserli@ethz.ch
       name: Lilian Gasser
-    dataset: politicians
-    path: Ratsmitglieder_1848_DE_corr.xlsx
+    path: ../../../data/politicians/Ratsmitglieder_1848_DE_corr.xlsx
     url: ../Ratsmitglieder_1848_DE_corr.xlsx
 identifier: 25c2b603-477d-43ec-a051-f0e8f44a2400
 name: politicians

.renku/datasets/2b13b133de164322b12072993d08ce89/metadata.yml

+'@context':
+  added: http://schema.org/dateCreated
+  affiliation: scoro:affiliate
+  authors:
+    '@container': '@list'
+  created: http://schema.org/dateCreated
+  dcterms: http://purl.org/dc/terms/
+  dctypes: http://purl.org/dc/dcmitypes/
+  email: dcterms:email
+  files:
+    '@container': '@index'
+  foaf: http://xmlns.com/foaf/0.1/
+  identifier:
+    '@id': dctypes:Dataset
+    '@type': '@id'
+  name: dcterms:name
+  prov: http://www.w3.org/ns/prov#
+  scoro: http://purl.org/spar/scoro/
+  url: http://schema.org/url
+'@type': dctypes:Dataset
+authors: []
+created: 2019-02-05 12:19:12.309142
+files:
+  ../../../data/lists/dict_overlaps.pickle:
+    '@type': http://schema.org/DigitalDocument
+    added: 2019-02-05 12:19:12.416049
+    authors: []
+    path: ../../../data/lists/dict_overlaps.pickle
+    url: ../data_from_nlp-democracy/results_overlap/dict_overlaps.pickle
+identifier: 2b13b133-de16-4322-b120-72993d08ce89
+name: lists

.renku/datasets/a9e0cfbd26944a7b8eb927c789a6b49e/metadata.yml

+'@context':
+  added: http://schema.org/dateCreated
+  affiliation: scoro:affiliate
+  authors:
+    '@container': '@list'
+  created: http://schema.org/dateCreated
+  dcterms: http://purl.org/dc/terms/
+  dctypes: http://purl.org/dc/dcmitypes/
+  email: dcterms:email
+  files:
+    '@container': '@index'
+  foaf: http://xmlns.com/foaf/0.1/
+  identifier:
+    '@id': dctypes:Dataset
+    '@type': '@id'
+  name: dcterms:name
+  prov: http://www.w3.org/ns/prov#
+  scoro: http://purl.org/spar/scoro/
+  url: http://schema.org/url
+'@type': dctypes:Dataset
+authors:
+- '@type': dcterms:creator
+  affiliation: null
+  email: gasserli@ethz.ch
+  name: Lilian Gasser
+created: 2019-01-14 14:18:40.238479
+files:
+  ../../../data/train_NER/1891_20026449_corrected_german.txt:
+    '@type': http://schema.org/DigitalDocument
+    added: 2019-02-07 12:38:59.620749
+    authors:
+    - '@type': dcterms:creator
+      affiliation: null
+      email: gasserli@ethz.ch
+      name: Lilian Gasser
+    path: ../../../data/train_NER/1891_20026449_corrected_german.txt
+    url: /home/lili/Desktop/1891_20026449_corrected_german.txt
+  ../../../data/train_NER/20190109_train_NER.tar.gz:
+    '@type': http://schema.org/DigitalDocument
+    added: 2019-01-14 14:22:34.457485
+    authors:
+    - '@type': dcterms:creator
+      affiliation: null
+      email: gasserli@ethz.ch
+      name: Lilian Gasser
+    path: ../../../data/train_NER/20190109_train_NER.tar.gz
+    url: ../20190109_train_NER.tar.gz
+  ../../../data/train_NER/20190116_train_NER_french.tar.gz:
+    '@type': http://schema.org/DigitalDocument
+    added: 2019-01-16 09:17:45.682598
+    authors:
+    - '@type': dcterms:creator
+      affiliation: null
+      email: gasserli@ethz.ch
+      name: Lilian Gasser
+    path: ../../../data/train_NER/20190116_train_NER_french.tar.gz
+    url: /home/lili/NLP_DemocraSci/20190116_train_NER_french.tar.gz
+identifier: a9e0cfbd-2694-4a7b-8eb9-27c789a6b49e
+name: train_NER

.renku/refs/datasets/AB

+../../datasets/67ee08facf874ee592a927022e65ab7b/metadata.yml
\ No newline at end of file

.renku/refs/datasets/lists

+../../datasets/2b13b133de164322b12072993d08ce89/metadata.yml
\ No newline at end of file

.renku/refs/datasets/politicians

+../../datasets/25c2b603477d43eca051f0e8f44a2400/metadata.yml
\ No newline at end of file

.renku/refs/datasets/politicians-yearly-csvs

+../../datasets/02eb93f040474fe0b1172cfa0f3b15e1/metadata.yml
\ No newline at end of file

.renku/refs/datasets/train_NER

+../../datasets/a9e0cfbd26944a7b8eb927c789a6b49e/metadata.yml
\ No newline at end of file

.renku/workflow/006dba80c770426183c68cf2eae67332_python.cwl

+arguments: []
+baseCommand:
+- python
+class: CommandLineTool
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default:
+      class: File
+      path: ../../src/python/run_correct_meta.py
+    inputBinding:
+      position: 1
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_2:
+    default:
+      class: File
+      path: ../../data/AB/1956/01_rawmeta.tar.gz
+    inputBinding:
+      position: 2
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_3:
+    default: data/AB/1956/03_correctedmeta.tar.gz
+    inputBinding:
+      position: 3
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: string
+outputs:
+  output_0:
+    outputBinding:
+      glob: $(inputs.input_3)
+    streamable: false
+    type: File
+permanentFailCodes: []
+requirements:
+- class: InlineJavascriptRequirement
+- class: InitialWorkDirRequirement
+  listing:
+  - entry: '$({"listing": [], "class": "Directory"})'
+    entryname: data/AB/1956
+    writable: true
+successCodes: []
+temporaryFailCodes: []

.renku/workflow/009a3dd30e014deab9475064b9478059_python.cwl

+arguments: []
+baseCommand:
+- python
+class: CommandLineTool
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default:
+      class: File
+      path: ../../src/python/run_collect_information.py
+    inputBinding:
+      position: 1
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_2:
+    default:
+      class: File
+      path: ../../data/AB/1976/05_annotatedxml.tar.gz
+    inputBinding:
+      position: 2
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_3:
+    default:
+      class: File
+      path: ../../data/AB/1976/03_correctedmeta.tar.gz
+    inputBinding:
+      position: 3
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_4:
+    default: data/AB/1976/06_collectedinformation.pickle
+    inputBinding:
+      position: 4
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: string
+outputs:
+  output_0:
+    outputBinding:
+      glob: $(inputs.input_4)
+    streamable: false
+    type: File
+permanentFailCodes: []
+requirements:
+- class: InlineJavascriptRequirement
+- class: InitialWorkDirRequirement
+  listing:
+  - entry: '$({"listing": [], "class": "Directory"})'
+    entryname: data/AB/1976
+    writable: true
+successCodes: []
+temporaryFailCodes: []

.renku/workflow/0145cac7beac4379b50ec0049916dea6_python.cwl

+arguments: []
+baseCommand:
+- python
+class: CommandLineTool
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default:
+      class: File
+      path: ../../src/python/run_extract_discussions.py
+    inputBinding:
+      position: 1
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_2:
+    default:
+      class: File
+      path: ../../data/politicians/yearly_pickles/1971_MPs.pickle
+    inputBinding:
+      position: 2
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_3:
+    default:
+      class: File
+      path: ../../data/AB/1971/04_correctedxml.tar.gz
+    inputBinding:
+      position: 3
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_4:
+    default:
+      class: File
+      path: ../../data/AB/1971/03_correctedmeta.tar.gz
+    inputBinding:
+      position: 4
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_5:
+    default:
+      class: File
+      path: ../../data/lists/not_names.txt
+    inputBinding:
+      position: 5
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_6:
+    default:
+      class: File
+      path: ../../data/lists/dict_overlaps.pickle
+    inputBinding:
+      position: 6
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_7:
+    default: data/AB/1971/05_annotatedxml.tar.gz
+    inputBinding:
+      position: 7
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: string
+outputs:
+  output_0:
+    outputBinding:
+      glob: $(inputs.input_7)
+    streamable: false
+    type: File
+permanentFailCodes: []
+requirements:
+- class: InlineJavascriptRequirement
+- class: InitialWorkDirRequirement
+  listing:
+  - entry: '$({"listing": [], "class": "Directory"})'
+    entryname: data/AB/1971
+    writable: true
+successCodes: []
+temporaryFailCodes: []