Merge pull request #47 from Deuchnord/e2e-tests

Add E2E tests

.editorconfig

@@ -1,3 +1,5 @@
+root = true
+
 [*]
 charset = utf-8
 end_of_line = lf

.github/workflows/e2e.yml

@@ -0,0 +1,24 @@
+name: E2E tests
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+
+    - name: Prepare tests
+      run: |
+        pip install -U setuptools pip
+
+    - name: E2E tests
+      run: |
+        export ENVIRONMENT="CI"
+        bash .scripts/tests-e2e.sh

.scripts/tests-e2e.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+VERSION=$(grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' kosmorrolib/version.py)
+PYTHON_BIN=$(command -v python)
+PIP_BIN=$(command -v pip)
+
+if python3 --version > /dev/null; then
+    PYTHON_BIN=$(command -v python3)
+    PIP_BIN=$(command -v pip3)
+fi
+
+failures=''
+
+function fail() {
+    failures="$failures\n\n - $1\n\n$2"
+}
+
+function run() {
+    eval "$1" &> /tmp/output.txt
+    return $?
+}
+
+function canRun() {
+    if [[ "$1" != "" && "$1" != "$ENVIRONMENT" ]]; then
+        return 1
+    fi
+
+    return 0
+}
+
+# Asserts that command $1 has finished with sucess
+# $1: the command to run
+function assertSuccess() {
+    if ! canRun "$2"; then
+        echo -n 'I'
+        return
+    fi
+
+    run "$1"
+    returned=$?
+
+    if [ $returned -ne 0 ]; then
+      fail "Failed asserting that command '$1' finishes with success, returned status $returned." "$(cat /tmp/output.txt)"
+      echo -n 'F'
+      return
+    fi
+
+    echo -n '.'
+}
+
+# Asserts that command $1 has finished with sucess
+# $1: the command to run
+function assertFailure() {
+    if ! canRun "$2"; then
+        echo -n 'I'
+        return
+    fi
+
+    run "$1"
+    returned=$?
+
+    if [ $returned -eq 0 ]; then
+      fail "Failed asserting that command '$1' finishes with failure." "$(cat /tmp/output.txt)"
+      echo -n 'F'
+      return
+    fi
+
+    echo -n '.'
+}
+
+echo
+echo "==== RUNNING E2E TESTS ===="
+echo
+
+# Create the package and install it
+assertSuccess "$PYTHON_BIN setup.py sdist"
+assertSuccess "$PIP_BIN install dist/kosmorro-$VERSION.tar.gz" "CI"
+
+assertSuccess kosmorro
+assertSuccess "kosmorro --latitude=50.5876 --longitude=3.0624"
+assertSuccess "kosmorro --latitude=50.5876 --longitude=3.0624 -d 27 -m 1 -y 2020"
+assertSuccess "kosmorro --latitude=50.5876 --longitude=3.0624 -d 27 -m 1 -y 2020 --format=json"
+assertFailure "kosmorro --latitude=50.5876 --longitude=3.0624 -d 27 -m 1 -y 2020 --format=pdf"
+# Missing dependencies, should fail
+assertFailure "kosmorro --latitude=50.5876 --longitude=3.0624 -d 27 -m 1 -y 2020 --format=pdf -o /tmp/document.pdf"
+
+assertSuccess "sudo apt-get install -y texlive" "CI"
+assertSuccess "$PIP_BIN install latex" "CI"
+
+# Dependencies installed, should not fail
+assertSuccess "kosmorro --latitude=50.5876 --longitude=3.0624 -d 27 -m 1 -y 2020 --format=pdf -o /tmp/document.pdf"
+
+if [ "$failures" != "" ]; then
+    echo -e "\n$failures"
+    exit 2
+fi
+
+echo -e "\n\n==== TESTS RAN SUCCESSFULLY 👍 ===="

CONTRIBUTING.md

@@ -70,8 +70,31 @@ Kosmorro's source code follows the major coding standards of Python (PEPs). Befo
 
 ### Testing the code
 
-Sadly, Kosmorro currently has a poor code coverage, but feel free to add new unit tests to enhance its stability.
-Any new Pull requests should have at least one new unit test that checks the new development is correct.
+There are two kinds of tests on this project: **unit tests** and **end-to-end tests** (sometimes abbreviated to _E2E tests_). 
+
+#### Unit tests
+
+[Unit tests](https://en.wikipedia.org/wiki/Unit_testing) check that every little piece of code (any _unit_) does exactly what it is supposed to do. They have several advantages, like proving that new things in the codebase works exactly as they should, and making sure that future changes done later won't break them.
+ 
+Kosmorro's unit tests use Python's official `unittest` module. They live in the `/test` folder of the repository. Whenever you write a new feature or a bug fix, please write the unit tests that will make sure it works.
+You can also run them by invoking the following command:
+
+```shell
+pipenv run python -m unittest test
+```
+
+Note: there are currently some memory leaks in the unit tests, making the result quite difficult to read. I am working to fix this.
+If you have troubles reading them, feel free to ask.
+
+#### End-to-end tests
+
+If unit tests are really good at checking that every individual parts of code work well, they fail at checking that so does the _whole_ program as a finished product. That is the job of end-to-end tests.
+
+The goal here is to make sure that if you install Kosmorro from scratch, it will work without any problem. If a mandatory dependency has not been installed, or if something goes wrong in the main program (which is not possible to unit test), the E2E tests will fail.
+
+Kosmorro's 2E2 tests are a Bash script living in the `/.scripts/tests-e2e.sh` file. You should only add tests here if you add new ways to interact with Kosmorro itself (e.g. adding a new argument in the command line).
+
+Fore security purpose, it is not recommended running E2E tests locally, because some tests need the root privilege to pass (e.g. installing optional dependencies). They are run on the CI.
 
 ### Commiting
 

MANIFEST.in

@@ -1 +1,2 @@
 recursive-include kosmorrolib/locales *
+recursive-include kosmorrolib/assets *

kosmorrolib/main.py

@@ -84,7 +84,7 @@ def main():
     elif not selected_dumper.is_file_output_needed():
         print(output)
     else:
-        print(_('Selected output format needs an output file (--output).'))
+        print(colored(_('Selected output format needs an output file (--output).'), color='red'))
         return 1
 
     return 0