add textract

.gitignore

@@ -0,0 +1,4 @@
+*.pyc
+.idea/
+.vscode
+.mypy_cache

Jenkinsfile

@@ -1,8 +1,9 @@
-String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1-5" : ""
+String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1,3,5" : ""
 
 pipeline {
   agent { label 'ephemeral-linux' }
   options {
+    // The Build GPU stage depends on the image from the Push CPU stage
     disableConcurrentBuilds()
   }
   triggers {
@@ -14,12 +15,13 @@ pipeline {
     GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim()
     GIT_COMMIT_SUMMARY = "`<https://github.com/Kaggle/docker-python/commit/${GIT_COMMIT}|${GIT_COMMIT_SHORT}>` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}"
     SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernelops\"; else echo \"#builds\"; fi").trim()
+    PRETEST_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"ci-pretest\"; else echo \"${GIT_BRANCH}-pretest\"; fi").trim()
+    STAGING_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"staging\"; else echo \"${GIT_BRANCH}-staging\"; fi").trim()
   }
 
   stages {
-    stage('Docker Build') {
+    stage('Docker CPU Build') {
       steps {
-        slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
         sh '''#!/bin/bash
           set -exo pipefail
 
@@ -28,9 +30,19 @@ pipeline {
       }
     }
 
-    stage('Test Image') {
+    stage('Push CPU Pretest Image') {
+      steps {
+        sh '''#!/bin/bash
+          set -exo pipefail
+
+          date
+          ./push ${PRETEST_TAG}
+        '''
+      }
+    }
+
+    stage('Test CPU Image') {
       steps {
-        slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} test image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
         sh '''#!/bin/bash
           set -exo pipefail
 
@@ -40,22 +52,94 @@ pipeline {
       }
     }
 
-    stage('Push Image') {
+    stage('Push CPU Image') {
       steps {
-        slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} pushing image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
         sh '''#!/bin/bash
           set -exo pipefail
 
           date
-          ./push staging
+          ./push ${STAGING_TAG}
         '''
       }
     }
+
+    stage('Docker GPU Build') {
+      // A GPU is not required to build this image. However, in our current setup,
+      // the default runtime is set to nvidia (as opposed to runc) and there
+      // is no option to specify a runtime for the `docker build` command.
+      //
+      // TODO(rosbo) don't set `nvidia` as the default runtime and use the
+      // `--runtime=nvidia` flag for the `docker run` command when GPU support is needed.
+      agent { label 'ephemeral-linux-gpu' }
+      steps {
+        sh '''#!/bin/bash
+          set -exo pipefail
+          docker image prune -f # remove previously built image to prevent disk from filling up
+          ./build --gpu --base-image-tag ${STAGING_TAG} | ts
+        '''
+      }
+    }
+
+    stage('Push GPU Pretest Image') {
+      agent { label 'ephemeral-linux-gpu' }
+      steps {
+        sh '''#!/bin/bash
+          set -exo pipefail
+
+          date
+          ./push --gpu ${PRETEST_TAG}
+        '''
+      }
+    }
+
+    stage('Test GPU Image') {
+      agent { label 'ephemeral-linux-gpu' }
+      steps {
+        sh '''#!/bin/bash
+          set -exo pipefail
+
+          date
+          ./test --gpu
+        '''
+      }
+    }
+
+    stage('Push GPU Image') {
+      agent { label 'ephemeral-linux-gpu' }
+      steps {
+        sh '''#!/bin/bash
+          set -exo pipefail
+
+          date
+          ./push --gpu ${STAGING_TAG}
+        '''
+      }
+    }
+
+    stage('Package Versions') {
+      parallel {
+        stage('CPU Diff') {
+          steps {
+            sh '''#!/bin/bash
+            ./diff
+          '''
+          }
+        }
+        stage('GPU Diff') {
+          agent { label 'ephemeral-linux-gpu' }
+          steps {
+            sh '''#!/bin/bash
+            ./diff --gpu
+          '''
+          }
+        }
+      }
+    }
   }
 
   post {
     failure {
-      slackSend color: 'danger', message: "*<${env.BUILD_URL}console|${JOB_NAME} failed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
+      slackSend color: 'danger', message: "*<${env.BUILD_URL}console|${JOB_NAME} failed>* ${GIT_COMMIT_SUMMARY} @kernels-backend-ops", channel: env.SLACK_CHANNEL
     }
     success {
       slackSend color: 'good', message: "*<${env.BUILD_URL}console|${JOB_NAME} passed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL

README.md

@@ -1,60 +1,88 @@
-**Kaggle Kernels** allow users to run scripts against our competitions and datasets without having to download data or set up their environment. Here's [an example](https://www.kaggle.com/devinanzelmo/d/devinanzelmo/dota-2-matches/setting-up-a-prediction-problem-dota-2):
+[Kaggle Kernels](https://www.kaggle.com/kernels) allow users to run a Python Notebook in the cloud against our competitions and datasets without having to download data or set up their environment.
 
-![example script](http://i.imgur.com/yrWycNA.png)
+This repository includes our Dockerfiles for building the [CPU-only](Dockerfile) and [GPU](gpu.Dockerfile) image that runs Python Kernels on Kaggle. 
 
-This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://registry.hub.docker.com/u/kaggle/python/) the Docker image on Dockerhub.
+Our Python Docker images are stored on Google Container Registry at:
+
+* CPU-only: [gcr.io/kaggle-images/python](https://gcr.io/kaggle-images/python)
+* GPU: private for now, we will make it public soon.
+
+Note: The base image for the GPU image is our CPU-only image. The [gpu.Dockerfile](gpu.Dockerfile) adds a few extra layers to install GPU related libraries and packages (cuda, libcudnn, pycuda etc.) and reinstall packages with specific GPU builds (torch, tensorflow and a few mores).
 
 ## Getting started
 
 To get started with this image, read our [guide](http://blog.kaggle.com/2016/02/05/how-to-get-started-with-data-science-in-containers/) to using it yourself, or browse [Kaggle Kernels](https://www.kaggle.com/kernels) for ideas.
 
-## Requesting new features
+## Requesting new packages
 
-**We welcome pull requests** if there are any packages you'd like to add!
+First, evaluate whether installing the package yourself in your own Kernels suits your needs. See [guide](https://github.com/Kaggle/docker-python/wiki/Missing-Packages).
 
-We can merge your request quickly if you check that it builds correctly. Here's how to do that.
+If you the first step above doesn't work for your use case, [open an issue](https://github.com/Kaggle/docker-python/issues/new) or a [pull request](https://github.com/Kaggle/docker-python/pulls).
 
-Start by running this image on your system:
+## Opening a pull request
 
-```
-me@my-computer:/home$ docker run --rm -it kaggle/python
-root@d72b81a003e1:/# 
-```
+1. Update the *Dockerfile*
+    1. For changes specific to the GPU image, update the [gpu.Dockerfile](gpu.Dockerfile).
+    1. Otherwise, update the [Dockerfile](Dockerfile).
+1. Follow the instructions below to build a new image.
+1. Add tests for your new package. See this [example](https://github.com/Kaggle/docker-python/blob/master/tests/test_fastai.py).
+1. Follow the instructions below to test the new image.
+1. Open a PR on this repo and you are all set!
 
-Then follow the package's installation instructions for a Linux system. It could be as simple as installing via Pip:
+## Building a new image
 
-```
-root@d72b81a003e1:/# pip install coolpackage
-Collecting coolpackage
-[...etc...]
+```sh
+./build
 ```
 
-Once that's done, check that you can import it correctly. (Sometimes, if a package is missing a dependency, it throws an error when you try to import it.)
+Flags:
 
-```
-root@d72b81a003e1:/# python
-Python 3.5.2 |Anaconda 4.2.0 (64-bit)| (default, Jul  2 2016, 17:53:06)
-[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)] on linux
-Type "help", "copyright", "credits" or "license" for more information.
->>> import coolpackage
->>>
-```
+* `--gpu` to build an image for GPU.
+* `--use-cache` for faster iterative builds.
 
-Once that's working, add the necessary lines to our [Dockerfile](https://github.com/Kaggle/docker-python/blob/master/Dockerfile). (In this case, that would mean adding `pip install coolpackage` to the last section.)
+## Testing a new image
 
-Next run the build:
+A suite of tests can be found under the `/tests` folder. You can run the test using this command:
 
+```sh
+./test
 ```
-./build
-```
 
-Finally run the tests:
+Flags:
+
+* `--gpu` to test the GPU image.
+
+## Running the image
 
+For the CPU-only image:
+
+```sh
+# Run the image built locally:
+docker run --rm -it kaggle/python-build /bin/bash
+# Run the pre-built image from gcr.io
+docker run --rm -it gcr.io/kaggle-images/python /bin/bash
 ```
-./tests
+
+For the GPU image:
+
+```sh
+# Run the image built locally:
+docker run --runtime nvidia --rm -it kaggle/python-gpu-build /bin/bash
+# Run the image pre-built image from gcr.io
+# TODO: Our GPU images are not yet publicly available.
 ```
 
-Then submit your pull request, and you're all set!
+To ensure your container can access the GPU, follow the instructions posted [here](https://github.com/Kaggle/docker-python/issues/361#issuecomment-448093930).
+
+## Tensorflow custom pre-built wheel
+
+A Tensorflow custom pre-built wheel is used mainly for:
+
+* Faster build time: Building tensorflow from sources takes ~1h. Keeping this process outside the main build allows faster iterations when working on our Dockerfiles.
 
+Building Tensorflow from sources:
 
+* Increase performance: When building from sources, we can leverage CPU specific optimizations
+* Is required: Tensorflow with GPU support must be built from sources
 
+The [Dockerfile](tensorflow-whl/Dockerfile) and the [instructions](tensorflow-whl/README.md) can be found in the [tensorflow-whl folder/](tensorflow-whl/).

build

@@ -1,11 +1,61 @@
 #!/bin/bash
 set -e
 
-# Default behavior is to do everything from scratch.
-# The --use-cache option is useful if you're iterating on a broken build.
-if [[ "$1" ==  "--use-cache" ]]; then
-    docker build --rm -t kaggle/python-build .
-else
-    docker pull continuumio/anaconda3:latest
-    docker build --rm --no-cache -t kaggle/python-build .
-fi
+usage() {
+cat << EOF
+Usage: $0 [OPTIONS]
+Build a new Python Docker image.
+
+Options:
+    -g, --gpu                 Build an image with GPU support.
+    -c, --use-cache           Use layer cache when building a new image.
+    -b, --base-image-tag TAG  Base image tag. Defaults to value defined in DOCKERFILE.
+EOF
+}
+
+CACHE_FLAG='--no-cache'
+DOCKERFILE='Dockerfile'
+IMAGE_TAG='kaggle/python-build'
+BUILD_ARGS=''
+
+while :; do
+    case "$1" in 
+        -h|--help)
+            usage
+            exit
+            ;;
+        -g|--gpu)
+            IMAGE_TAG='kaggle/python-gpu-build'
+            DOCKERFILE='gpu.Dockerfile'
+            ;;
+        -c|--use-cache)
+            CACHE_FLAG=''
+            ;;
+        -b|--base-image-tag)
+            if [[ -z $2 ]]; then
+                usage
+                printf 'ERROR: No TAG specified after the %s flag.\n' "$1" >&2
+                exit
+            fi
+            BUILD_ARGS="--build-arg BASE_TAG=$2"
+            shift # skip the flag value
+            ;;
+        -?*)
+            usage
+            printf 'ERROR: Unknown option: %s\n' "$1" >&2
+            exit
+            ;;
+        *)            
+            break
+    esac
+
+    shift
+done
+
+readonly CACHE_FLAG
+readonly DOCKERFILE
+readonly IMAGE_TAG
+readonly BUILD_ARGS
+
+set -x
+docker build --rm --pull $CACHE_FLAG -t "$IMAGE_TAG" -f "$DOCKERFILE" $BUILD_ARGS .

clean-layer.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# This scripts should be called at the end of each RUN command
+# in the Dockerfiles.
+#
+# Each RUN command creates a new layer that is stored separately.
+# At the end of each command, we should ensure we clean up downloaded
+# archives and source files used to produce binary to reduce the size
+# of the layer.
+set -e
+set -x
+
+# Delete files that pip caches when installing a package.
+rm -rf /root/.cache/pip/*
+# Delete old downloaded archive files 
+apt-get autoremove -y
+# Delete downloaded archive files
+apt-get clean
+# Delete source files used for building binaries
+rm -rf /usr/local/src/*
+# Delete conda downloaded tarballs
+conda clean -y --tarballs