Move doc to top-level

ci/travis_script_python.sh

@@ -66,6 +66,7 @@ if [ "$ARROW_TRAVIS_PYTHON_DOCS" == "1" ] && [ "$PYTHON_VERSION" == "3.6" ]; the
         numpydoc \
         sphinx \
         breathe \
+        recommonmark \
         doxygen \
         sphinx_rtd_theme
 fi
@@ -195,7 +196,7 @@ if [ "$ARROW_TRAVIS_PYTHON_DOCS" == "1" ] && [ "$PYTHON_VERSION" == "3.6" ]; the
   pushd ../cpp/apidoc
   doxygen
   popd
-  cd doc
+  cd ../doc
   sphinx-build -q -b html -d _build/doctrees -W source _build/html
 fi
 

dev/gen_apidocs/create_documents.sh

@@ -91,7 +91,7 @@ popd
 pushd arrow/python
 python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
     --with-plasma --with-parquet --inplace
-python setup.py build_sphinx -s doc/source
+python setup.py build_sphinx -s ../doc/source
 mkdir -p ../site/asf-site/docs/python
 rsync -r doc/_build/html/ ../site/asf-site/docs/python
 popd

doc/requirements.txt

@@ -2,5 +2,6 @@ breathe
 ipython
 matplotlib
 numpydoc
+recommonmark
 sphinx
 sphinx_rtd_theme

doc/source/conf.py

@@ -30,6 +30,8 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
+from recommonmark.parser import CommonMarkParser
+
 import os
 import sys
 
@@ -77,14 +79,25 @@ napoleon_use_rtype = False
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
-breathe_projects = {"arrow_cpp": "../../../cpp/apidoc/xml"}
+breathe_projects = {"arrow_cpp": "../../cpp/apidoc/xml"}
 breathe_default_project = "arrow_cpp"
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+
+class CMWithLicenseParser(CommonMarkParser):
+
+    def parse(self, inputstring, document):
+        # Strip the license header from the MarkDown file. Sadly recommmark cannot handle this.
+        inputstring = inputstring[inputstring.index('>'):]
+        return super(CMWithLicenseParser, self).parse(inputstring, document)
+
+source_suffix = ['.rst', '.md']
+
+source_parsers = {
+   '.md': CMWithLicenseParser
+}
 
 autosummary_generate = True
 

doc/source/format/Layout.md

@@ -17,7 +17,7 @@
   under the License.
 -->
 
-# Arrow: Physical memory layout
+# Physical memory layout
 
 ## Definitions / Terminology
 

doc/source/format/README.md

@@ -17,7 +17,7 @@
   under the License.
 -->
 
-## Arrow specification documents
+# Arrow specification documents
 
 Currently, the Arrow specification consists of these pieces:
 

doc/source/index.rst

@@ -25,14 +25,18 @@ hardware. It also provides computational libraries and zero-copy streaming
 messaging and interprocess communication.
 
 .. toctree::
+   :maxdepth: 1
    :caption: Memory Format
 
-   format
+   format/README
+   format/Guidelines
+   format/Layout
+   format/Metadata
+   format/IPC
 
 .. toctree::
    :maxdepth: 2
    :caption: Languages
 
-   language_interop
    cpp/index
    python/index

python/README.md

@@ -79,8 +79,8 @@ and look for the "custom options" section.
 ### Building the documentation
 
 ```bash
-pip install -r doc/requirements.txt
-python setup.py build_sphinx -s doc/source
+pip install -r ../doc/requirements.txt
+python setup.py build_sphinx -s ../doc/source
 ```
 
 [2]: https://github.com/apache/arrow/blob/master/python/doc/source/development.rst