Planet Python

or four - I'll follow up on that · real challenge

Francois Dion
@f_dion
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

%matplotlib inline # needed for jupyter notebooks

plt.rcParams['figure.figsize']=(20,10) # set the figure size
plt.style.use('fivethirtyeight') # using the fivethirtyeight matplotlib theme

sales = pd.read_csv('examples/sales.csv') # Read the data in
sales.Date = pd.to_datetime(sales.Date) #set the date column to datetime
sales.set_index('Date', inplace=True) #set the index to the date column

# now the hack for the multi-colored bar chart: 
# create fiscal year dataframes covering the timeframes you are looking for. In this case,
# the fiscal year covered October - September.
# --------------------------------------------------------------------------------
# Note: This should be set up as a function, but for this small amount of data,
# I just manually built each fiscal year. This is not very pythonic and would
# suck to do if you have many years of data, but it isn't bad for a few years of data. 
# --------------------------------------------------------------------------------

fy10_all = sales[(sales.index >= '2009-10-01') & (sales.index < '2010-10-01')]
fy11_all = sales[(sales.index >= '2010-10-01') & (sales.index < '2011-10-01')]
fy12_all = sales[(sales.index >= '2011-10-01') & (sales.index < '2012-10-01')]
fy13_all = sales[(sales.index >= '2012-10-01') & (sales.index < '2013-10-01')]
fy14_all = sales[(sales.index >= '2013-10-01') & (sales.index < '2014-10-01')]
fy15_all = sales[(sales.index >= '2014-10-01') & (sales.index < '2015-10-01')]

# Let's build our plot

fig, ax1 = plt.subplots()
ax2 = ax1.twinx()  # set up the 2nd axis
ax1.plot(sales.Sales_Dollars) #plot the Revenue on axis #1

# the next few lines plot the fiscal year data as bar plots and changes the color for each.
ax2.bar(fy10_all.index, fy10_all.Quantity,width=20, alpha=0.2, color='orange')
ax2.bar(fy11_all.index, fy11_all.Quantity,width=20, alpha=0.2, color='gray')
ax2.bar(fy12_all.index, fy12_all.Quantity,width=20, alpha=0.2, color='orange')
ax2.bar(fy13_all.index, fy13_all.Quantity,width=20, alpha=0.2, color='gray')
ax2.bar(fy14_all.index, fy14_all.Quantity,width=20, alpha=0.2, color='orange')
ax2.bar(fy15_all.index, fy15_all.Quantity,width=20, alpha=0.2, color='gray')

ax2.grid(b=False) # turn off grid #2

ax1.set_title('Monthly Sales Revenue vs Number of Items Sold Per Month')
ax1.set_ylabel('Monthly Sales Revenue')
ax2.set_ylabel('Number of Items Sold')

# Set the x-axis labels to be more meaningful than just some random dates.
labels = ['FY 2010', 'FY 2011','FY 2012', 'FY 2013','FY 2014', 'FY 2015']
ax1.axes.set_xticklabels(labels)
from lomond import WebSocket
websocket = WebSocket('wss://echo.example.org')
for event in websocket:
    if event.name == 'text':
        websocket.send_text(event.text)

from lomond import WebSocket
from lomond.persist import persist
websocket = WebSocket('wss://echo.example.org')
for event in persist(websocket):
    if event.name == 'text':
        websocket.send_text(event.text)

def merge_lists(list_one, label_one, list_two, label_two):
    """ Generate to return the earliest dated log entry from two lists """
    one = next(list_one)
    two = next(list_two)
    while one or two:
        if one['time'] < two['time']:
            yield label_one, one
            one = next(list_one)
        else:
            yield label_two, two
            two = next(list_two)

def merge_lists_refactor(lists, labels):
    objects = list(map(next, lists))
    while any(x is not None for x in objects):
        i, obj = min(enumerate(objects), 
                     key=lambda x: (x[1]['time'], x[0]))
        yield labels[i], obj
        objects[i] = next(lists[i])

    objects = list(map(next, lists))

    while any(x is not None for x in objects):

    [x is not None for x in objects]

        i, obj = min(enumerate(objects), 
                          key=lambda x: (x[1]['time'], x[0]))

        enumerate(objects)

        i, obj = min(<generated list>, key=<comparison_function>)

(index, {'time': <datetime obj>, 'other_stuff':<otherstuff>})

        (x[1]['time'], x[0]))

(index, {'time': <datetime obj>, 'other_stuff':<otherstuff>})

(<datetime obj>, index)

        i, obj = min(enumerate(objects), key=lambda x: (x[1]['time'], x[0]))

        yield labels[i], obj
        objects[i] = next(lists[i])

list(heapq.merge(a,b, key=lambda....))

def create_td(td_name):
    request_id = str(uuid.uuid4())
    body = {'name': td_name}
    return DRIVE.teamdrives().create(body=body,
            requestId=request_id, fields='id').execute().get('id')

def add_user(td_id, user, role='commenter'):
    body = {'type': 'user', 'role': role, 'emailAddress': user}
    return DRIVE.permissions().create(body=body, fileId=td_id,
            supportsTeamDrives=True, fields='id').execute().get('id')

def create_td_folder(td_id, folder):
    body = {'name': folder, 'mimeType': FOLDER_MIME, 'parents': [td_id]}
    return DRIVE.files().create(body=body,
            supportsTeamDrives=True, fields='id').execute().get('id')

def import_csv_to_td_folder(folder_id, fn, mimeType):
    body = {'name': fn, 'mimeType': mimeType, 'parents': [folder_id]}
    return DRIVE.files().create(body=body, media_body=fn+'.csv',
            supportsTeamDrives=True, fields='id').execute().get('id')

FOLDER_MIME = 'application/vnd.google-apps.folder'
SOURCE_FILE = 'inventory' # on disk as 'inventory.csv'
SHEETS_MIME = 'application/vnd.google-apps.spreadsheet'

td_id = create_td('Corporate shared TD')
print('** Team Drive created')
perm_id = add_user(td_id, 'email@example.com')
print('** User added to Team Drive')
folder_id = create_td_folder(td_id, 'Manufacturing data')
print('** Folder created in Team Drive')
file_id = import_csv_to_td_folder(folder_id, SOURCE_FILE, SHEETS_MIME)
print('** CSV file imported as Google Sheets in Team Drives folder')

$ python3 td_demo.py
** Team Drive created
** User added to Team Drive
** Folder created in Team Drive
** CSV file imported as Google Sheets in Team Drives folder

from __future__ import print_function
import uuid

from apiclient import discovery
from httplib2 import Http
from oauth2client import file, client, tools

SCOPES = 'https://www.googleapis.com/auth/drive'
store = file.Storage('storage.json')
creds = store.get()
if not creds or creds.invalid:
    flow = client.flow_from_clientsecrets('client_secret.json', SCOPES)
    creds = tools.run_flow(flow, store)
DRIVE = discovery.build('drive', 'v3', http=creds.authorize(Http()))

def create_td(td_name):
    request_id = str(uuid.uuid4()) # random unique UUID string
    body = {'name': td_name}
    return DRIVE.teamdrives().create(body=body,
            requestId=request_id, fields='id').execute().get('id')

def add_user(td_id, user, role='commenter'):
    body = {'type': 'user', 'role': role, 'emailAddress': user}
    return DRIVE.permissions().create(body=body, fileId=td_id,
            supportsTeamDrives=True, fields='id').execute().get('id')

def create_td_folder(td_id, folder):
    body = {'name': folder, 'mimeType': FOLDER_MIME, 'parents': [td_id]}
    return DRIVE.files().create(body=body,
            supportsTeamDrives=True, fields='id').execute().get('id')

def import_csv_to_td_folder(folder_id, fn, mimeType):
    body = {'name': fn, 'mimeType': mimeType, 'parents': [folder_id]}
    return DRIVE.files().create(body=body, media_body=fn+'.csv',
            supportsTeamDrives=True, fields='id').execute().get('id')

FOLDER_MIME = 'application/vnd.google-apps.folder'
SOURCE_FILE = 'inventory' # on disk as 'inventory.csv'... CHANGE!
SHEETS_MIME = 'application/vnd.google-apps.spreadsheet'

td_id = create_td('Corporate shared TD')
print('** Team Drive created')
perm_id = add_user(td_id, 'email@example.com') # CHANGE!
print('** User added to Team Drive')
folder_id = create_td_folder(td_id, 'Manufacturing data')
print('** Folder created in Team Drive')
file_id = import_csv_to_td_folder(folder_id, SOURCE_FILE, SHEETS_MIME)
print('** CSV file imported as Google Sheets in Team Drives folder')

pip install fbprophet
conda install -c conda-forge fbprophet
pip install pystan
from fbprophet import Prophet
import numpy as np
import pandas as pd
sales_df = pd.read_csv('../examples/retail_sales.csv')
sales_df['y_orig'] = sales_df['y'] # to save a copy of the original data..you'll see why shortly. 
# log-transform y
sales_df['y'] = np.log(sales_df['y'])
model = Prophet() #instantiate Prophet
model.fit(sales_df); #fit the model with your dataframe
Disabling weekly seasonality. Run prophet with weekly_seasonality=True to override this.
future_data = model.make_future_dataframe(periods=6, freq = 'm')
<span class="n">forecast_data</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">future_data</span><span class="p">)</span>
forecast_data[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail()
itmodel.plot(forecast_data)
model.plot_components(forecast_data)
forecast_data_orig = forecast_data # make sure we save the original forecast data
forecast_data_orig['yhat'] = np.exp(forecast_data_orig['yhat'])
forecast_data_orig['yhat_lower'] = np.exp(forecast_data_orig['yhat_lower'])
forecast_data_orig['yhat_upper'] = np.exp(forecast_data_orig['yhat_upper'])
model.plot(forecast_data_orig)
sales_df['y_log']=sales_df['y'] #copy the log-transformed data to another column
sales_df['y']=sales_df['y_orig'] #copy the original data to 'y'
model.plot(forecast_data_orig)
1
2
3
def multiply(a, b):
    product = a * b
    return product

1
2
3
4
5
def multiply(a, b):
    factor1 = a
    factor2 = b
    result = a * b
    return result

1
2
3
class A(object):
    def __init__(self):
        self._note = "a note"

1
2
3
4
class B(A):
    def __init__(self):
        super().__init__()
        self._note = "private state for B()"

1
2
3
4
class Bprime(object):
    def __init__(self, a):
        for var in dir(a):
            setattr(self, var, getattr(a, var))

# seafoam/__init__.py
from pathlib import Path

def get_path():
    """
    Shortcut for users whose theme is not next to their pelicanconf.py.

    Returns:
        str: file path to folder containing theme
    """
    # Theme directory is defined as our parent directory
    return str(Path(__file__).resolve().parent)

# pelicanconf.py

import seafoam
THEME = seafoam.get_path()
BOOTSTRAP_THEME = 'seafoam'

# ... other settings

pelican -s publishconf.py --fatal=warnings

# .travis.yml

# build using Python 3.6
language: python
python:
  - 3.6
# Cache Dependencies
cache: pip
# we need two Linux packages for the optimize_images plugin
addons:
  apt:
    packages:
    - optipng
    - libjpeg-progs

# install dependencies
install:
  - pip install --upgrade pip
  - pip install -r requirements.txt
# build our blog
script:
  - invoke publish_carefully

# deploy to GitHub pages
deploy:
  provider: pages
  local_dir: ../blog.minchin.ca-master/
  target-branch: master
  skip_cleanup: true
  github_token: $GITHUB_TOKEN  # Set in travis-ci.org dashboard
  # only deploy on "main" branch
  on:
    branch: pelican

>>> import ipaddress
>>> my_ip = ipaddress.ip_address('192.168.100.10')
>>> my_ip
IPv4Address('192.168.100.10')

# All 32 binary bits can be used to create an IPv4 address:
>>> ipaddress.ip_address(0b11000000101010000110010000001010)
IPv4Address('192.168.100.10')

# The decimal value of the 32 bit binary number can also be used:
>>> ipaddress.ip_address(3232261130)
IPv4Address('192.168.100.10')

# As can the hexadecimal value of the 32 bits:
>>> ipaddress.ip_address(0xC0A8640A)
IPv4Address('192.168.100.10')

# An ip_interface object is used to represent IP addressing
# for a host or router interface, including the CIDR:
>>> my_ip = ipaddress.ip_interface('192.168.100.10/24')
>>> my_ip
IPv4Interface('192.168.100.10/24')

# This method translates the CIDR into a mask as would normally
# be used on a host or router interface
>>> my_ip.netmask
IPv4Address('255.255.255.0')

# Creates an ip_network object. The IPv4 address and CIDR must be
# a valid network address, the first address in an address range:
>>> ipaddress.ip_network('192.168.100.0/24')
IPv4Network('192.168.100.0/24')

# Python will throw an exception if the address used is not
# a valid network address. In the following, ".10" is a host address
# not a valid network address ident cation, which is ".0":
>>> ipaddress.ip_network('192.168.100.10/24')
ValueError: "192.168.100.10/24 has host bits set"

# If the network address needs to be calculated,
# use the strict=False option. This will calculate and populate
# the ip_network object with the network rather than the
# interface address:
>>> my_ip = ipaddress.ip_interface('192.168.100.10/24')
>>> my_ip
IPv4Interface('192.168.100.10/24')

>>> my_ip_net = ipaddress.ip_network(my_ip, strict=False)
>>> my_ip_net
IPv4Network('192.168.100.0/24')

# Create an IPv6 Address Object for a Global Address:
>>> ipaddress.ip_address('2001:db8:abcd:100::1')
IPv6Address('2001:db8:abcd:100::1')

# Create an IPv6 Address Object for a link-local address:
>>> ipaddress.ip_address('fe80::1')
IPv6Address('fe80::1')

# Creates an IP Interface Object for a Global Address:
>>> ipaddress.ip_interface('2001:db8:abcd:100::1/64')
IPv6Interface('2001:db8:abcd:100::1/64')

# Creates an IP Interface Object for a Link-local Address:
ipaddress.ip_interface('fe80::1/64')
IPv6Interface('fe80::1/64')

# Creates an IP Network Object for a Global Address:
>>> myIPv6net = ipaddress.ip_network('2001:db8:abcd:100::/64')
>>> myIPv6net
IPv6Network('2001:db8:abcd:100::/64')

# Creates an IP Network Object for a Link-local Address:
>>> myIPv6 = ipaddress.ip_network('fe80::/64')
>>> myIPv6
IPv6Network('fe80::/64')

<html>
    <head>
        <title>My Simple Webpage</title>
    </head>
    <body>
        <p>I can write anything I want here. Add some <b>bold</b> or
        <i>italics</i> text. For everything else, I'll send you to
        <a href="https://www.google.com">Google</a>!</p>
    </body>
</html>

Title: My super title
Date: 2010-12-03 10:20
Modified: 2010-12-05 19:30
Category: Python
Tags: pelican, publishing
Slug: my-super-post
Authors: Alexis Metaireau, Conan Doyle
Summary: Short version for index and feeds

This is the content of my super blog post.

![Image Alt Text](/path/to/image.jpg)

```python
    # python code block
    print("A literal block directive explicitly marked as python code")
```

A sentence with links to [Wikipedia](http://www.wikipedia.org/) and the
[Linux kernel archive](http://www.kernel.org/).

My super title
##############

:date: 2010-10-03 10:20
:modified: 2010-10-04 18:40
:tags: thats, awesome
:category: yeah
:slug: my-super-post
:authors: Alexis Metaireau, Conan Doyle
:summary: Short version for index and feeds

This will be turned into :abbr:`HTML (HyperText Markup Language)`.

.. image:: /path/to/image.jpg

.. code-block:: python
    :classprefix: pgcss
    :linenos: table
    :linenostart: 153

   print("A literal block directive explicitly marked as python code")

A sentence with links to Wikipedia_ and the `Linux kernel archive`_.

.. _Wikipedia: http://www.wikipedia.org/
.. _Linux kernel archive: http://www.kernel.org/

<html>
    <head>
        <title>My super title</title>
        <meta name="tags" content="thats, awesome" />
        <meta name="date" content="2012-07-09 22:28" />
        <meta name="modified" content="2012-07-10 20:14" />
        <meta name="category" content="yeah" />
        <meta name="authors" content="Alexis Métaireau, Conan Doyle" />
        <meta name="summary" content="Short version for index and feeds" />
    </head>
    <body>
        This is the content of my super blog post.
    </body>
</html>

<html>
    <head>
        <title>{% block title %}{% endblock %}</title>
    </head>
    <body>
        {% block body %}{% endblock %}
    </body>
</html>

{% extends "layout.html" %}

{% block title -%}
    All Users
{%- endblock %}

{% block body %}
    <ul>
    {% for user in users %}
        <li><a href="{{ user.url }}">{{ user.username }}</a></li>
    {% endfor %}
    </ul>
{% endblock %}

{% extends "base.html" %}

{% block title %}Archives - {{ SITENAME }}{% endblock %}

{% block breadcrumbs %}
    {% if DISPLAY_BREADCRUMBS %}
    <ol class="breadcrumb">
        <li><a href="{{ SITEURL }}" title="{{ SITENAME }}"><i class="fa fa-home fa-lg"></i></a></li>
        <li class="active">Archives</li>
    </ol>
    {% endif %}
{% endblock %}

{% block content %}
    <section id="content">
        <h1>Archives for {{ SITENAME }}</h1>
        <div id="archives">
            {%- set last_year = None -%}
            {%- set last_month = None -%}
            {%- set last_day = None -%}

            {% for article in dates %}
                <div class="row">
                    {% if article.date.year != last_year %}
                        {% if last_year != None -%}
                            <div class="archives-spacer col-xs-12">&nbsp;</div>
                            <div class="archives-spacer col-xs-12">&nbsp;</div>
                        {%- endif %}
                        <div class="archives-date archives-year col-xs-4 col-sm-2 col-sm-offset-2">
                            <a name="{{ article.date.year }}"></a>
                            {{- article.date.year -}}
                        </div>
                        <div class="archives-title col-xs-7 col-sm-6">
                            &nbsp;
                        </div>
                        {%- set last_year = article.date.year -%}
                        {%- set last_month = None -%}
                        {%- set last_day = None -%}
                    {% endif %}
                    {% if article.date.month != last_month %}
                        {% if last_month != None -%}
                            <div class="archives-spacer col-xs-12">&nbsp;</div>
                        {%- endif %}
                        <div class="archives-date archives-month col-xs-4 col-sm-2 col-sm-offset-2">
                        <a name="{{ article.date.year }}-{{ article.date.month }}"></a>
                            {{- article.date|strftime('%B') -}}
                        </div>
                        <div class="archives-title col-xs-7 col-sm-6">
                            &nbsp;
                        </div>
                        {%- set last_month = article.date.month -%}
                        {%- set last_day = None -%}
                    {% endif %}

                    <div class="archives-date categories-timestamp col-xs-4 col-sm-2 col-sm-offset-2">
                        {%- if last_day != article.date.day %}
                            <time datetime="{{ article.date.isoformat() }}">{{ article.date | strftime('%a %-d') }}</time>
                        {% else -%}
                            &nbsp;
                        {%- endif -%}
                    </div>
                    <div class="archives-title col-xs-7 col-sm-6">
                        <a href="{{ SITEURL }}/{{ article.url }}">{{ article.title }}</a>
                        {% if article.subtitle %}<br />{{ article.subtitle }}{% endif %}
                    </div>
                    {%- set last_day = article.date.day %}
                </div>
            {% endfor %}
        </div>
    </section>
{% endblock %}

C:> pip install pelican markdown
        [...]
C:> pelican-quickstart

yourproject/
    ├── content/             # articles and pages go here
    ├── output/
    ├── develop_server.sh
    ├── fabfile.py
    ├── Makefile
    ├── pelicanconf.py       # Main settings file
    └── publishconf.py       # Settings to use when ready to publish

yourproject/
    ├── content/
    │   ├── article1.rst
    │   ├── cat/
    │   │   └── article2.md
    │   ├── images
    │   │   └── han.jpg
    │   ├── pages/
    │   │   ├── about.md
    │   │   └── (other articles)
    │   └── (other articles)
    ├── output
    ├── develop_server.sh
    ├── fabfile.py
    ├── Makefile
    ├── pelicanconf.py       # Main settings file
    └── publishconf.py       # Settings to use when ready to publish

Title: The second article
Date: 2012-12-01 10:02

See below intra-site link examples in Markdown format.

[a link relative to the current file]({filename}../article1.rst)
[a link relative to the content root]({filename}/article1.rst)

![Alt Text]({filename}/images/han.jpg)

#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = 'Wm Minchin'
SITENAME = 'Introduction to Pelican'
SITEURL = ''

PATH = 'content'
ARTICLE_PATHS = ['posts']
PAGE_PATHS = ['.']

TIMEZONE = 'America/Edmonton'

DEFAULT_LANG = 'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 10

#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

# This file is only used if you use `make publish` or
# explicitly specify it as your config file.

import os
import sys
sys.path.append(os.curdir)
from pelicanconf import *

SITEURL = 'http://minchin.ca'
RELATIVE_URLS = False

FEED_ALL_ATOM = 'feeds/all.atom.xml'
CATEGORY_FEED_ATOM = 'feeds/%s.atom.xml'

DELETE_OUTPUT_DIRECTORY = True

# Following items are often useful when publishing

#DISQUS_SITENAME = ""
#GOOGLE_ANALYTICS = ""

@ECHO off
@ECHO Delete old JRNL export
del S:\Documents\jrnl-pelican\content\*.md

:: Export from JRNL
jrnl --export yaml -o S:\Documents\jrnl-pelican\content\
jrnl dayone --export yaml -o S:\Documents\jrnl-pelican\content\

:: Export from PRJCT
prjct project_entry S:\Documents\jrnl-pelican\content\all_projects.md

:: Pelican
cd S:\Documents\jrnl-pelican
@ECHO Running Pelican...
pelican -s publishconf.py
robocopy S:\Documents\jrnl-pelican\output\ \\myserver\web\jrnl\ /MIR /nfl /ndl

title: Kurt Vonnegut's 8 Tips on How to Write a Great Story
date: 2012-10-15 14:04
stared: False
tags: whitespace, bookmarks, writing, writing_tips

**The Atlantic Home** --
Saturday, July 7, 2012 --
By Maria Popova --
[Source](http://www.theatlantic.com/entertainment/archive/2012/04/kurt-vonneguts-8-tips-on-how-to-write-a-great-story/255401/)

Why you should be cruel to your readers

The year of reading more and writing better is well underway with writing advice 
the likes of David Ogilvy's 10 no-bullshit tips, Henry Miller's 11 commandments,
Jack Kerouac's 30 beliefs and techniques, John Steinbeck's 6 pointers, and
various invaluable insight from other great writers.

Now comes Kurt Vonnegut -- anarchist, Second Life dweller, imaginary interviewer
of the dead, sad soul -- with eight tips on how to write a good short story,
narrated by the author himself.

1. Use the time of a total stranger in such a way that he or she will not feel the time was wasted.
2. Give the reader at least one character he or she can root for.
3. Every character should want something, even if it is only a glass of water.
4. Every sentence must do one of two things—reveal character or advance the action.
5. Start as close to the end as possible.
6. Be a Sadist. No matter how sweet and innocent your leading characters, make awful things happen to them—in order that the reader may see what they are made of.
7. Write to please just one person. If you open a window and make love to the world, so to speak, your story will get pneumonia.
8. Give your readers as much information as possible as soon as possible. To hell with suspense. Readers should have such complete understanding of what is going on, where and why, that they could finish the story themselves, should cockroaches eat the last few pages.

#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

import re

import prjct
# requires  pymdown-extensions

AUTHOR = u'WM'
SITENAME = u'Jrnl Notebook'
SITEURL = ''

PATH = 'content'
CUSTOM_CSS = SITEURL + '_css/jrnl.css'

CACHE_CONTENT = True
LOAD_CACHE_CONTENT = True
CHECK_MODIFIED_METHOD = 'md5'   # default is the file's modified time, which is
                                # useless for us because we keep regenerating
                                # the source files

TIMEZONE = 'America/Edmonton'

DEFAULT_LANG = u'en'
DEFAULT_DATE = 'fs'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None

DEFAULT_PAGINATION = 10
TYPOGRIFY = True
TYPOGRIFY_IGNORE_TAGS = ['code', 'pre', 'tt']

STATIC_PATHS = ['images', '_css']

THEME = 'themes/pelican-bootstrap3'

FAVICON = SITEURL + 'images/favicon.ico'
FAVICON_IE = SITEURL + 'images/favicon.ico'
SITELOGO = SITEURL + 'images/favicon-48.png'

ELEVATOR = False

BOOTSTRAP_THEME = 'superhero'
USE_PAGER = True
BOOTSTRAP_NAVBAR_INVERSE = True
DISPLAY_TAGS_INLINE = True
DISPLAY_BREADCRUMBS = True

# see https://pythonhosted.org/Markdown/reference.html#markdown
# format required for Pelican 3.7
MARKDOWN = {
    'extension_configs': {
        # set linenums=True for line numbers
        # https://pythonhosted.org/Markdown/extensions/code_hilite.html
        'markdown.extensions.codehilite': {'css_class': 'highlight'},
        # on by default
        # see https://pythonhosted.org/Markdown/extensions/extra.html
        'markdown.extensions.extra': {},
        'markdown.extensions.meta': {},
        # https://facelessuser.github.io/pymdown-extensions/extensions/superfences/
        'pymdownx.superfences': {},
        # use single caret for superscript
        # use double caret for insertion (underline?)
        # https://facelessuser.github.io/pymdown-extensions/extensions/caret/
        'pymdownx.caret': {},
        # use single tildes for subscript
        # use double tildes for deletion
        # https://facelessuser.github.io/pymdown-extensions/extensions/tilde/
        'pymdownx.tilde': {},
        # https://pythonhosted.org/Markdown/extensions/smarty.html  
        'markdown.extensions.smarty': {},
        # so MathJax equations (always) make it through
        # https://facelessuser.github.io/pymdown-extensions/extensions/arithmatex/
        'pymdownx.arithmatex': {},
        # Auto-convert special symbols, like (tm) to ™
        # https://facelessuser.github.io/pymdown-extensions/extensions/smartsymbols/
        'pymdownx.smartsymbols': {},
    },
    'output_format': 'html5',
    'lazy_ol': False,
}

# A list of metadata fields containing reST/Markdown content to be parsed and translated to HTML.
FORMATTED_FIELDS = ['title', 'summary']

PLUGIN_PATHS = ['../GitHub/pelican-plugins', ]
PLUGINS = ['tipue_search',
           'tag_cloud',
           'prjct.titlecase',  # titlecase Jinja filter
           ]

TAG_CLOUD_STEPS = 4
TAG_CLOUD_MAX_ITEMS = 100

prjct_release = str(prjct.__version__)
p = re.compile('\d+\.\d+(\.\d+)?')
prjct_versionmatch = p.match(prjct_release)
prjct_version = prjct_versionmatch.group()

PRJCT = True
PRJCT_TODO, PRJCT_DONE = prjct.todo_export.to_html_dicts(prjct.config.load())
PRJCT_PROJECTS = prjct.todo_export.project_list()
PRJCT_DESC = prjct.descriptions.to_html_dict(prjct.config.load(), MD_EXTENSIONS)
PRJCT_VERSION = prjct_version
PRJCT_URL = prjct.__url__

PAGE_URL =              '{slug}/'
PAGE_SAVE_AS =          '{slug}/index.html'
TAGS_URL =              'tags/'
TAGS_SAVE_AS =          'tags/index.html'
TAG_URL =               'tags/{slug}/'
TAG_SAVE_AS =           'tags/{slug}/index.html'
CATEGORIES_URL =        'categories/'
CATEGORIES_SAVE_AS =    'categories/index.html'
CATEGORY_URL =          'categories/{slug}/'
CATEGORY_SAVE_AS =      'categories/{slug}/index.html'
AUTHORS_URL =           'authors/'
AUTHORS_SAVE_AS =       'authors/index.html'
AUTHOR_URL =            'authors/{slug}/'
AUTHOR_SAVE_AS =        'authors/{slug}/index.html'

# {date:%b}  gives short month in words (i.e. 'Apr')
ARTICLE_URL =           'posts/{date:%Y}/{date:%-m}/{date:%-d}/{slug}/'
ARTICLE_SAVE_AS =       'posts/{date:%Y}/{date:%-m}/{date:%-d}/{slug}/index.html'
DAY_ARCHIVE_URL =       'posts/{date:%Y}/{date:%-m}/{date:%-d}/'
DAY_ARCHIVE_SAVE_AS =   'posts/{date:%Y}/{date:%-m}/{date:%-d}/index.html'
MONTH_ARCHIVE_URL =     'posts/{date:%Y}/{date:%-m}/'
MONTH_ARCHIVE_SAVE_AS = 'posts/{date:%Y}/{date:%-m}/index.html'
YEAR_ARCHIVE_URL =      'posts/{date:%Y}/'
YEAR_ARCHIVE_SAVE_AS =  'posts/{date:%Y}/index.html'

ARCHIVES_URL =          'posts/'
ARCHIVES_SAVE_AS =      'posts/index.html'

PRJCT_URL =             'prjct/'
PRJCT_SAVE_AS =         'prjct/index.html'

DIRECT_TEMPLATES = ['index', 'categories', 'authors', 'archives', 
                    'search', 'tags', '404', 'prjct']
PAGINATED_DIRECT_TEMPLATES = ['index', 'archives']

# Altered fabfile to use with Invoke
from invoke import run, task

import os
import zipfile
from build_js import build_js
from floating_rate_vs_rro_graph import make_graph

# Local path configuration (can be absolute or relative to fabfile)
env_deploy_path = 'output'
env_zipfile = 'burstenergy.zip'

@task
def clean():
    if os.path.isdir(env_deploy_path):
        run('rm -rf ' + env_deploy_path)
        run('mkdir ' + env_deploy_path)

@task
def build():
    run('pelican -s pelicanconf.py')
    build_js()
    make_graph()

@task
def rebuild():
    clean()
    build()

@task
def regenerate():
    run('start pelican -r -s pelicanconf.py')
    build_js()
    make_graph()

@task
def serve():
    run('cd ' + env_deploy_path + ' && start python -m http.server')

@task
def reserve():
    build()
    build_js()
    make_graph()
    serve()

@task
def preview():
    run('pelican -s publishconf.py')

@task
def build_zip():
    zipf = zipfile.ZipFile(env_zipfile, 'w', zipfile.ZIP_DEFLATED)
    for root, dirs, files in os.walk(env_deploy_path):
        for file in files:
            absfile = os.path.join(root, file)
            zfile = absfile[len(env_deploy_path)+len(os.sep):]  # relative path
            zipf.write(absfile, zfile)
    zipf.close()

@task
def zip():
    clean()
    build()
    build_zip()

@task
def develop():
    clean()
    regenerate()
    serve()

@task
def build_dieppe():
    run('pelican -s pelicanconf-dieppe.py')

@task
def less():
    run('lessc theme\\burst-energy\\less\\bootstrap.burst-energy.less > ' +
        env_deploy_path + '\\css\\style.css')
    #   lessc theme\burst-energy\less\bootstrap.burst-energy.less > output\css\style.css

@task
def take_screenshot():
    from ghost import Ghost

    url = "http://localhost:8000"
    gh = Ghost()

    # We create a new page
    page, page_name = gh.create_page()

    # We load the main page of ebay
    page_resource = page.open(url, wait_onload_event=True)

    # Save the image of the screen
    page.capture_to("burst-energy.png")

#!/usr/bin/env python3
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

'''
This builds the JavaScript files that are used by the MySecure content.

Build the site first, then this will put the JS files in the output directory.
'''

def build_js():
    from pelicanconf import OUTPUT_PATH
    from bs4 import BeautifulSoup
    import os
    import re
    import codecs

    'configuration'
    'form file'
    FORM_FILE = (os.path.dirname(os.path.realpath(__file__)) + os.sep +
                 OUTPUT_PATH + os.sep)
    FORM_FILE += 'mysecure' + os.sep + 'signup.html'

    'output directory'
    JS_PATH = (os.path.dirname(os.path.realpath(__file__)) + os.sep +
               OUTPUT_PATH + os.sep + 'js')

    'Placeholder text'
    placeholder = 'PLACEHOLDER'

    "test existence of 'form file'"

    with open(FORM_FILE, "r") as FORM_FILE_HANDLE:
        soup = BeautifulSoup(FORM_FILE_HANDLE, "lxml")

    "create `title.js`"
    "document title"
    with open(JS_PATH + os.sep + 'title.js', "w") as TITLE_JS:
        print("var titleblock = ''", file=TITLE_JS)
        print("titleblock += '{}'".format(soup.title), file=TITLE_JS)
        print("", file=TITLE_JS)
        # ToDo: Select the title dynamically,
        #                               based on the file it's called from"

        "meta -- viewport"
        viewport = soup.find('meta', {'name': 'viewport'})
        print("titleblock += '{}'".format(viewport), file=TITLE_JS)
        print("", file=TITLE_JS)

        print("document.write(titleblock);", file=TITLE_JS)

    "create `header.js` and `footerinfo.js`"
    on_header = True
    my_re2 = re.compile('</?body>')
    my_re4 = re.compile(placeholder)
    on_header = True
    with codecs.open(JS_PATH + os.sep + 'header.js', "w", "utf-8") \
            as HEADER_JS:
        with codecs.open(JS_PATH + os.sep + 'footerinfo.js', "w", "utf-8") \
                as FOOTER_JS:
            print("var headerblock = ''", file=HEADER_JS)
            print("var footerblock = ''", file=FOOTER_JS)
            for line in str(soup.find('body')).split('\n'):
                line2 = my_re2.sub('', line)
                if my_re4.search(line2):
                    on_header = False
                    line2 = ''
                if on_header:
                    print("headerblock += '{}'".format(line2), file=HEADER_JS)
                else:
                    print("footerblock += '{}'".format(line2), file=FOOTER_JS)
            print("document.write(headerblock)", file=HEADER_JS)
            print("document.write(footerblock)", file=FOOTER_JS)

if __name__ == "__main__":
    build_js()

#   This generated the graph show the comparision between the RRO and our
#   floating rate.
#   This graph is displayed on https://www.burstenergy.ca/rates/floating/

Months = ['2014-01', '2014-02', '2014-03', '2014-04', '2014-05', '2014-06', 
          '2014-07', '2014-08', '2014-09', '2014-10', '2014-11', '2014-12',
          '2015-01', '2015-02', '2015-03', '2015-04', '2015-05', '2015-06',
          '2015-07', '2015-08', '2015-09', '2015-10', '2015-11', '2015-12',
          '2016-01', '2016-02', '2016-03', '2016-04']

# both floating and RRO for Edmonton
RRO =    [ 8.689,     7.471,     6.991,     6.986,      8.922,    5.995,
           7.198,     8.021,     7.954,     8.737,      7.127,    7.545,
           7.302,     6.583,     5.431,     5.832,      4.337,    4.089,
           6.140,     5.813,     5.387,     5.498,      5.212,    5.489,
           5.304,     4.753,     4.521,     3.65]

Floating = [ 6.149,  11.730,     5.903,     4.438,      7.328,    5.909,
            15.801,   6.239,     3.710,     4.011,      5.378,    4.049,
             5.001,   4.831,     3.294,     3.282,      7.461,   13.218,
             3.692,   4.952,     3.327,     3.412,      3.403,    3.333,
             3.441,   2.907,     2.617,     2.503]

# NGX Spot Prices
NGX =    [3.8412,    5.4560,    5.1446,    4.4361,     4.4374,   4.3885,
          4.2005,    3.8446,    3.8608,    3.7947,     3.7084,   3.6218,
          3.0539,    2.7301,    2.7503,    2.5161,     2.5477,   2.5729,
          2.5976,    2.7783,    2.7699,    2.6217,     2.4256,   2.3347,
          2.3324,    2.0379,    1.5752,    1.3116]

NG_margin = 0.90

def make_graph(Months_Plotted=12):
    import os
    from pathlib import Path
    from statistics import mean

    import matplotlib.pyplot as plt
    import matplotlib.dates as mdates
    from pylab import savefig
    from datetime import datetime
    # for nicer colours, importing is enough
    import seaborn as sns

    """Convert *Months* to numbers"""
    for i in range(len(Months)):
        year, month = map(int, Months[i].split('-'))
        Months[i] = mdates.date2num(datetime(year, month, 1))

    """drop last month on RRO if we have more data than for the floating"""
    if len(RRO) > len(Floating):
        my_RRO = RRO[:-1]
        my_Months = Months[:-1]
    else:
        my_RRO = RRO
        my_Months = Months
    my_Floating = Floating
    my_NG = [i + NG_margin for i in NGX]

    if len(my_Months) > Months_Plotted:
        my_Floating = my_Floating[-Months_Plotted:]
        my_RRO = my_RRO[-Months_Plotted:]
        my_Months = my_Months[-Months_Plotted:]
        my_NG = my_NG[-Months_Plotted:]

    # averages
    average_RRO = [mean(my_RRO) for i in my_Months]
    average_Floating = [mean(my_Floating) for i in my_Months]
    average_NG = [mean(my_NG) for i in my_Months]

    """Now let's plot!"""
    # make the final image 800px wide
    fig = plt.figure(dpi=72, figsize=(800/72/1.19, (800*1/2)/72/1.19))
    graph = fig.add_subplot(111)
    # Plot the two lines
    line_RRO = graph.plot(my_Months, my_RRO, sns.xkcd_rgb["pale red"], label="RRO")
    line_Burst = graph.plot(my_Months, my_Floating, sns.xkcd_rgb["medium green"], label="Burst Energy")

    # x axis -- list the months on the 1st
    graph.set_xticks(my_Months)
    graph.set_xticklabels([mdates.num2date(date).strftime("%b '%y") for date in my_Months])
    # y axis -- min is 0, label
    plt.ylim(ymin=0)
    plt.ylabel("\u00A2 / kWh, for Edmonton")
    # add legend
    graph.legend()

    # current file location (this script file)
    p = Path(os.path.realpath(__file__))
    # drop script file name
    p = p.parents[0]
    # location of where we want to save the graph
    p = p / "floating-rate-vs-rro.png"

    savefig(str(p), bbox_inches='tight')
    # plt.show()

    """Now let's plot (Natural Gas!"""
    # make the final image 800px wide
    fig = plt.figure(dpi=72, figsize=(800/72/1.19, (800*1/2)/72/1.19))
    graph = fig.add_subplot(111)
    # Plot the two lines
    line_Burst = graph.plot(my_Months, my_NG, sns.xkcd_rgb["medium green"], label="Burst Energy")

    # x axis -- list the months on the 1st
    graph.set_xticks(my_Months)
    graph.set_xticklabels([mdates.num2date(date).strftime("%b '%y") for date in my_Months])
    # y axis -- min is 0, label
    plt.ylim(ymin=0, ymax=6)
    plt.ylabel("$ / GJ, for Alberta")
    # add legend
    graph.legend()

    # current file location (this script file)
    p = Path(os.path.realpath(__file__))
    # drop script file name
    p = p.parents[0]
    # location of where we want to save the graph
    p = p / "natural-gas.png"

    savefig(str(p), bbox_inches='tight')
    # plt.show()

def averages(Months_Plotted=12):
    import statistics

    """drop last month on RRO if we have more data than for the floating"""
    if len(RRO) > len(Floating):
        my_RRO = RRO[:-1]
        my_Months = Months[:-1]
    else:
        my_RRO = RRO
        my_Months = Months
    my_Floating = Floating
    my_NG = [i + NG_margin for i in NGX]

    if len(my_Months) > Months_Plotted:
        my_Floating = my_Floating[-Months_Plotted:]
        my_RRO = my_RRO[-Months_Plotted:]
        my_Months = my_Months[-Months_Plotted:]
        my_NG = my_NG[-Months_Plotted:]

    ave_RRO = statistics.mean(my_RRO)
    ave_Floating = statistics.mean(my_Floating)
    ave_NG = statistics.mean(my_NG)

    print("For the last {} months, the average is:".format(Months_Plotted))
    print("{}For the RRO,           {:.3f} ¢/kWh".format("  ", ave_RRO))
    print("{}For the Floating rate, {:.3f} ¢/kWh".format("  ", ave_Floating))
    print("{}For natural gas,      ${:.2f} /GJ".format("  ", ave_NG))

if __name__ == "__main__":
    make_graph(12)
    averages(6)
    averages(12)

#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = 'Burst Energy'
SITENAME = 'Burst Energy'
SITEURL = 'https://www.burstenergy.ca'    # used by the sitemap generator
WEB_URL = ''    # "SITEURL" as used by the templates (so for the published web pages)
CANONICAL_SITEURL = SITEURL

PATH = 'content'
OUTPUT_PATH = 'output'

TIMEZONE = 'America/Edmonton'

DEFAULT_LANG = u'en'
DEFAULT_DATE = 'fs'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Theme
THEME = 'theme/burst-energy'
CUSTOM_CSS = 'css/burst-energy.css'
BANNER_SUBTITLE = 'Simple Electricity'
HIDE_SIDEBAR = True
BANNER = BANNER_ALL_PAGES = False
TYPOGRIFY = True
COPY_DATE = '2014-15'
BOOTSTRAP_NAVBAR_INVERSE = False
SITELOGO = WEB_URL + 'images/burst-energy-logo-sol-white-230.png'
# SITELOGO_WIDTH = '110px'
SITELOGO_HEIGHT = '25px'
FOOTER_LOGO = WEB_URL + 'images/burst-energy-logo-sol-230.png'
FOOTER_LOGO_WIDTH = '220px'
HIDE_SITENAME = True
BOOTSTRAP_THEME = 'burst-energy'
THEME_COLOR = "#2C3E50"

FAVICON = WEB_URL + 'images/favicon.png'
FAVICON_IE = WEB_URL + 'favicon.ico'
TOUCHICON = WEB_URL + 'images/apple-touch-icon.png'
# PYGMENTS_STYLE =

GOOGLE_ANALYTICS_UNIVERSAL = 'UA-49197109-1'
GOOGLE_ANALYTICS_UNIVERSAL_PROPERTY = 'burstenergy.ca'

DISPLAY_PAGES_ON_MENU = False
DISPLAY_CATEGORIES_ON_MENU = False
DISPLAY_ARCHIVES_ON_MENU = False
MENU_ITEMS_ONE = (('News', WEB_URL + '/news/', None),
                  ('Our Rates', WEB_URL + '/rates/', None),
                  ('Electricity 101', WEB_URL + '/electricity-101/', None),
                  )
MENU_ITEMS_TWO = (('Sign Up', WEB_URL + '/mysecure/signup.html', 'glyphicon glyphicon-ok-circle'),
                  ('My Account', WEB_URL + '/mysecure/portal.html', 'fa fa-sign-in'),
                  )
INDEX_SAVE_AS = "news/index.html"
DISPLAY_ARTICLE_INFO_ON_INDEX = True

# Link Structure
PAGE_URL = '{slug}/'
PAGE_SAVE_AS = '{slug}/index.html'
ARTICLE_URL = 'news/{date:%Y}/{slug}/'
ARTICLE_SAVE_AS = 'news/{date:%Y}/{slug}/index.html'
YEAR_ARCHIVE_SAVE_AS = 'news/{date:%Y}/index.html'
CATEGORIES_SAVE_AS = 'category/index.html'
AUTHORS_SAVE_AS = 'author/index.html'

STATIC_PATHS = ['images',
                'js',
                'favicon.ico',
                'browserconfig.xml',
                'robots.txt', ]

# Contact Info
CONTACT_WEB_EMAIL = "webteam@burstenergy.ca"
CONTACT_PHONE = "(780) 665-9918"
C_RETAIL_FAX = "(403) 265-7290"
C_CORPORATE_EMAIL = "william@burstenergy.ca"
FOOTER_ADDRESS = "Suite 200, 1316 9<sup>th</sup> Avenue SE<br />Calgary, Alberta T2G&nbsp;0T3"
SOCIAL = (('facebook', 'https://www.facebook.com/BurstEnergyCA'),
          ('twitter', 'http://twitter.com/BurstEnergy'),
          )
FOOTER_LINKS = (('About Us',    WEB_URL + '/about/'),
                ('FAQ',         WEB_URL + '/faq/'),
                ('Glossary',    WEB_URL + '/gloss/'),
                ('Quick Links', WEB_URL + '/quicklinks/'),
                ('Privacy',     WEB_URL + '/privacy/'),
                ('Legal',       WEB_URL + '/legal/'),
                )

# Plugins
PLUGIN_PATHS = ['S:\\Documents\\GitHub\\pelican-plugins', 'C:\\Users\\User\\Documents\\GitHub\\pelican-plugins']
PLUGINS = ['pelican_alias',
           'assets',
           'extended_sitemap',
           'neighbors']

USE_ASSETS = True
ASSETS_CSS = True
ASSETS_JS = False

SITEMAP = {
    "format": "xml",
}

EXTENDED_SITEMAP_PLUGIN = {
    'priorities': {
        'index': 1.0,
        'articles': 0.5,
        'pages': 0.8,
        'others': 0.4
    },
    'changefrequencies': {
        'index': 'daily',
        'articles': 'weekly',
        'pages': 'weekly',
        'others': 'monthly',
    }
}

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
# RELATIVE_URLS = True

# Nav bar links are set expliciately above
# pages set to hidden don't show up in sitemap
# sitemap also uses the 'date' as last modified

#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'Wm. Minchin'
SITENAME = u'Minchin.ca'
SITEURL = 'http://minchin.ca'
SITE_ROOT_URL = SITEURL

TIMEZONE = 'America/Edmonton'

DEFAULT_LANG = u'en'

# Uncomment following line if you want document-relative URLs when developing
RELATIVE_URLS = True

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None

DEFAULT_PAGINATION = False

# static paths will be copied under the same name
STATIC_PATHS = ['images',
                '..\extras',
                'css',
                'projects\design',
                '..\.gitattributes',
                '..\.gitignore',
                '..\README.txt', ]

# A list of files to copy from the source to the destination
EXTRA_PATH_METADATA = {
    '../.gitattributes':                     {'path': '.gitattributes'},
    '../.gitignore':                         {'path': '.gitignore'},
    '../README.txt':                         {'path': 'README.txt'},
    # CNAME file tells GitHub Pages to display this site at minchin.ca
    '../extras/CNAME':                       {'path': 'CNAME'},
    '../extras/minchin.ico':                 {'path': 'favicon.ico'},
    '../extras/MTS_1v1.xlsm':                {'path': 'MTS_1v1.xlsm'},
    '../extras/TRB_Minchin.ca.XSL':          {'path': 'TRB_Minchin.ca.XSL'},
    '../extras/googlecbc66a9bfde8606b.html': {'path': 'googlecbc66a9bfde8606b.html'},
    '../extras/.nojekyll':                   {'path': '.nojekyll'},
    # prism is used by the blog for code highlighting
    '../extras/prism.js':                    {'path': 'js/prism.js'},
    '../extras/prism.css':                   {'path': 'css/prism.css'},
    }

# Custom settings
MARKUP = (('rst',
           'md',
           'markdown',
           'mkd',
           'mdown',))  # don't include htm and html files
READERS = {'html': None,
           'htm': None}
PATH = 'content'
OUTPUT_PATH = '../minchinweb.github.io-master/'

# Add Blog to sidebar
MENUITEMS = (('Blog',    'http://blog.minchin.ca/', 'fa fa-pencil'), )
DISPLAY_PAGES_ON_MENU = True

# disable Tags, etc
TAGS_SAVE_AS = ''
TAG_SAVE_AS = ''
CATEGORY_URL = ''
CATEGORY_SAVE_AS = ''
CATEGORIES_URL = ''
CATEGORIES_SAVE_AS = ''
ARTICLE_URL = ''
ARTICLE_SAVE_AS = ''
AUTHORS_URL = ''
AUTHORS_SAVE_AS = ''
ARCHIVES_URL = ''
ARCHIVES_SAVE_AS = ''
PAGE_URL = "{slug}/"
PAGE_SAVE_AS = "{slug}/index.html"

# Theme Related
TYPOGRIFY = True
THEME = 'themes/pelican-minchin-ca'
SITELOGO = 'images/MinchindotCA-200.png'
SITELOGO_SIZE = '100%'
PYGMENTS_STYLE = 'friendly'
DISPLAY_BREADCRUMBS = True
FAVICON = 'favicon.ico'
BOOTSTRAP_THEME = 'minchin-ca'
USE_OPEN_GRAPH = True
CUSTOM_CSS = 'css/minchin-ca.css'
DOCUTIL_CSS = False

GOOGLE_ANALYTICS_UNIVERSAL = 'UA-384291-3'
GOOGLE_ANALYTICS_UNIVERSAL_PROPERTY = 'minchin.ca'

# Plugins
PLUGIN_PATHS = ['../pelican-plugins', ]
PLUGINS = ['assets', ]

ASSET_CSS = False
ASSET_JS = False

SITEMAP = {
    "format": "xml",
}

# # Make things disappear
DISPLAY_CATEGORIES_ON_MENU = False
HIDE_SITENAME = True
HIDE_SIDEBAR = True
FEED_ALL_ATOM = False
FEED_ALL_RSS = False
GITHUB_USER = False
ADDTHIS_PROFILE = False
DISQUS_SITENAME = False
PDF_PROCESSOR = False

#!/usr/bin/python
# -*- coding: utf-8 -*-

'''Genealogy Uploader
v.3.2.3 - WM - January 7, 2016

This script serves to semi-automate the building and uploading of my
genealogy website. It is intended to be semi-interactive and run from the
command line.'''

import codecs
from datetime import date, datetime
import fileinput
import multiprocessing as mp
import os
from pathlib import Path
import re
import sys
import textwrap
import uuid
import webbrowser
import zipfile

from bs4 import BeautifulSoup
import colorama
from colorama import Fore, Style
from invoke import run, task
from joblib import Parallel, delayed
import requests
import winshell

import minchin.text

__version__ = '3.2.3'
colorama.init()

COPYRIGHT_START_YEAR = 1987
ADAM_LINK = "http://gigatrees.com"
ADAM_FOOTER = "<p><strong>Are we related?</strong> Are you a long lost cousin? Spotted an error here? This website remains a work-in-progress and I would love to hear from you. Drop me a line at minchinweb [at] gmail [dot] com.</p>"
INDENT = " "*4
GITHUB_FOLDER = Path("S:\Documents\GitHub\genealogy-gh-pages")
PHOTO_FOLER = Path("S:\Documents\Genealogy")
DOWNLOAD_FOLDER = Path("S:\Downloads\Chrome")
URL_ROOT = "http://minchin.ca/genealogy"
REPO_URL = "https://github.com/MinchinWeb/genealogy.git"
ADAM_PREFIX = 'william-minchin-gigatree-offline-'
TODAY_STR = '' + str(date.today().year)[2:] + str.zfill(str(date.today().month), 2) + str.zfill(str(date.today().day), 2)
GEDCOM_EXPECTED = 'William ' + TODAY_STR + '.ged'
USER_FOLDER = Path(os.path.expanduser('~'))
MY_GEDCOM = USER_FOLDER / 'Desktop' / GEDCOM_EXPECTED
start_time = datetime.now()
step_no = 0  # step counter
# folder where the script is saved
HERE_FOLDER = Path(os.path.dirname(os.path.realpath(__file__)))
WORKING_FOLDER = HERE_FOLDER  # current working directory
CONTENT_FOLDER = HERE_FOLDER / 'content' / 'pages'
adam_zip = ''               # set later
tracking_filename = ''      # set later

def addimage(image):
    '''Take the file listed in image, finds in my genealogy photo directory, and
    adds it to the GitHub folder.'''
    '''TO-DO: implement this!!'''
    pass

# multiple replacement
# from  http://stackoverflow.com/questions/6116978/python-replace-multiple-strings
#
# Usage:
# >>> replacements = (u"café", u"tea"), (u"tea", u"café"), (u"like", u"love")
# >>> print multiple_replace(u"Do you like café? No, I prefer tea.", *replacements)
# Do you love tea? No, I prefer café.
def multiple_replacer(*key_values):
    replace_dict = dict(key_values)
    replacement_function = lambda match: replace_dict[match.group(0)]
    pattern = re.compile("|".join([re.escape(k) for k, v in key_values]), re.M | re.I)
    return lambda string: pattern.sub(replacement_function, string)

def multiple_replace(string, *key_values):
    return multiple_replacer(*key_values)(string)

def get_adam_version():
    soup_file = open(str(CONTENT_FOLDER / 'names.html'), 'r')
    soup = BeautifulSoup(soup_file, "lxml")
    soup_file.close()
    return soup.find(True, "gt-version").get_text().encode('utf-8')  # 'Built by Adam 1.35.0.0' or the like

@task
def export_gedcom():
    '''Export from RootsMagic.'''
    global step_no
    global start_time
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Export from RootsMagic.")

    print("{}call the file {}{}{} and save it to the desktop".format(INDENT*2, Style.BRIGHT, GEDCOM_EXPECTED, Style.RESET_ALL))
    print("{}do not include LDS information".format(INDENT*2))
    print("{}no need to privatize individuals (at this step)".format(INDENT*2))
    if not minchin.text.query_yes_quit("{}Next?".format(INDENT), default="yes"):
        sys.exit()
    try:
        start_time = datetime.fromtimestamp(os.stat(MY_GEDCOM).st_ctime)
    except:
        print("{}Your file doesn't seem to exist. Exiting...".format(INDENT))

@task
def clean_gedcom():
    '''Cleaning up GEDCOM.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Cleaning up GEDCOM.")

    # replace image paths
    gedcom_file = open(str(MY_GEDCOM), 'r', encoding='utf-8')  # add failsafe is the fail doesn't exist yet or is still being written to
    subject = gedcom_file.read()
    gedcom_file.close()

    pattern = re.compile(r'S:\\Documents\\Genealogy\\([0-9]+[\.[a-z]+]*\.? )*', re.IGNORECASE)  # path start
    result = pattern.sub('images/', subject)
    pattern2 = re.compile(r'(images.*)\\')  # reverse slashes in rest of path
    result2 = pattern2.sub(r'\1/', result)
    result3 = pattern2.sub(r'\1/', result2)

    f_out = open(str(MY_GEDCOM), 'w', encoding='utf-8')
    f_out.write(result3)
    f_out.close()

@task
def upload_gedcom():
    '''Upload GEDCOM to Gigatree.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". The file is now ready to upload to Gigatrees.")

    webbrowser.open("http://gigatrees.com/toolbox/gigatree", new=2)
    print("{}log-in (using Facebook)".format(INDENT*2))
    print("{}now click 'generate report'".format(INDENT*2))
    # check to see if we're logged in
    # log in, if needed
    # discard old GEDCOM
    # upload new GEDCOM
    # run generator
    # download new output

@task
def check_images():
    '''Check which images have already been uploaded.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Checking images.")

    gedcom_file = open(str(MY_GEDCOM), 'r', encoding='utf-8')
    subject = gedcom_file.read()
    gedcom_file.close()

    missing_matches = []
    all_matches = []
    matches = 0
    wrapper = textwrap.TextWrapper(width=79, initial_indent=INDENT, subsequent_indent=INDENT*2)
    pattern_bad = re.compile("missing ")
    for match in re.findall(r'(images/.+\.(jpg|jpeg|png|gif|pdf))', subject, re.IGNORECASE):
        all_matches.append(match)

    all_matches = sorted(set(all_matches))  # remove duplicates and sort
    for match in all_matches:
        r = requests.head(URL_ROOT + "/" + str(match[0]), allow_redirects=True)
        if not r.status_code == requests.codes.ok:
            mytext = wrapper.fill("missing {} -> {}".format(str(r.status_code), match[0]))
            print(pattern_bad.sub(Fore.RED + Style.BRIGHT + "missing " + Style.RESET_ALL, mytext))
            missing_matches.append(match[0])
        else:
            matches += 1

    if len(missing_matches) == 0:
        print("{}{} images matching. No missing images.".format(INDENT, str(matches)))
    else:
        print("{}{} images matching.".format(INDENT, str(matches)))
        q_add_images = minchin.text.query_yes_no_all("{}{} missing images. Add them?".format(INDENT, str(len(missing_matches))), default="no")
        if q_add_images == 2:  # all
            for image in missing_matches:
                addimage(image)
        elif q_add_images == 1:  # yes
            for image in missing_matches:
                if minchin.text.querry_yes_no("{}Add {}?".format(INDENT*2, image), default="yes"):
                    addimage(image)
                    # TO-DO: implement this!
                else:
                    pass
        else:  # no
            pass

        # write missing images to a file
        f = open('missing-images.txt', 'w', encoding='utf-8')
        f.write("Genealogy Uploader, v.{}\n".format(str(__version__)))
        f.write('{}\n\n'.format(MY_GEDCOM))
        for missing in missing_matches:
            f.write('{}\n'.format(missing))
        f.close()

@task
def delete_old_output():
    '''Delete old Pelican output.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Deleting old Pelican output.")

    to_delete = []
    html_files = 0
    os.chdir(str(GITHUB_FOLDER))
    all_files = os.listdir(str(GITHUB_FOLDER))

    for filename in all_files:
        if filename == ('.git'):
            pass  # don't drop the GIT repo
        elif filename == ('images'):
            pass  # don't drop the image folder
        elif filename.endswith('.html'):
            html_files += 1
        else:
            to_delete.append(filename)

    counter = 0

    # delete HTML files
    run('del *.html -y')
    bar = minchin.text.progressbar(maximum=len(to_delete) + html_files)
    counter = html_files
    bar.update(counter)

    # delete everything else
    for my_file in to_delete:
        winshell.delete_file(my_file, no_confirm=True, allow_undo=False, silent=True)
        counter += 1
        bar.update(counter)
    print("\n{}{} files deleted.".format(INDENT*2, str(len(to_delete) + html_files)))

@task
def delete_old_adam():
    '''Delete old Gigatrees output.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Deleting old Gigatree output.")

    os.chdir(str(CONTENT_FOLDER))
    run('del *.* /q')

@task
def get_new_adam():
    '''Get new Gigatree output.'''
    # TO-DO: allow override of 'start time'
    global step_no
    global adam_zip
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Get new Gigatree output.")

    os.chdir(str(DOWNLOAD_FOLDER))
    gedcom_time = datetime.fromtimestamp(os.stat(str(MY_GEDCOM)).st_ctime)

    count_loops = 0
    while True:
        all_files = os.listdir(str(DOWNLOAD_FOLDER))
        for filename in all_files:
            if filename.startswith(ADAM_PREFIX) and filename.endswith(".zip"):
                if datetime.fromtimestamp(os.stat(filename).st_ctime) > gedcom_time:
                    adam_zip = filename
        if adam_zip != '' and os.stat(adam_zip).st_size > 1000:
            break
        count_loops += 1
        if count_loops > 60:
            if minchin.text.query_yes_quit("{}We've waited 30 minutes. Keep waiting?".format(INDENT), default="yes") is False:
                sys.exit()
            else:
                count_loops = 0
        else:
            minchin.text.wait(30)

    winshell.copy_file(adam_zip, str(CONTENT_FOLDER))

def step_unzip():
    # Test 1: 6:48.948 for 9,999 files
    # Test 2: 2:05.188204 for 11,1698 files
    global step_no
    start_time_local = datetime.now()
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Unzip new Gigatree output (Zipfile).")

    os.chdir(str(CONTENT_FOLDER))
    zf = zipfile.ZipFile(adam_zip)
    zf.extractall()
    zf.close()
    print(INDENT, datetime.now() - start_time_local)

def step_unzip_faster():
    # see http://dmarkey.com/wordpress/2011/10/15/python-zipfile-speedup-tips/
    # Test 1: 4:44.459 for 9,999 files
    # this doesn't appear to work on Python 3.5.1, says it's not a zip file
    global step_no
    start_time_local = datetime.now()
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Unzip new Gigatree output (Zipfile faster).")

    os.chdir(str(CONTENT_FOLDER))
    zf = zipfile.ZipFile(open(adam_zip, 'r'))
    zf.extractall()
    zf.close()
    print(INDENT, datetime.now() - start_time_local)

def step_unzip_czip():
    # Test 1: 4:46.109 for 9,999 files
    # Test 2: xx for 11,1698 files
    global step_no
    start_time_local = datetime.now()
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Unzip new Gigatree output (czip).")

    os.chdir(str(CONTENT_FOLDER))
    zf = czipfile.ZipFile(adam_zip)
    zf.extractall()
    zf.close()
    print(INDENT, datetime.now() - start_time_local)

@task
def step_unzip_7zip():
    # Test 1: 5:09.974 for 9,999 files
    # Test 2: 1:43.229726 for 11,1698 files
    global step_no
    start_time_local = datetime.now()
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Unzip new Gigatree output (7-zip).")

    os.chdir(str(CONTENT_FOLDER))
    run('"C:\\Program Files\\7-Zip\\7z.exe" e {} > nul'.format(adam_zip))
    print(INDENT, datetime.now() - start_time_local)

def step_unzip_infozip():
    # Test 2: 1:51.550671 for 11,1698 files
    global step_no
    start_time_local = datetime.now()
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Unzip new Gigatree output (unzip.exe).")

    os.chdir(str(CONTENT_FOLDER))
    run('C:\\bin\\unzip.exe {} > nul'.format(adam_zip))
    print(INDENT, datetime.now() - start_time_local)

@task
def unzip_adam():
    '''Unzip new Adam output.'''
    try:
        step_unzip_7zip()
    except:
        step_unzip()

@task
def php_to_html():
    '''Change any .php files to .html.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Rename all .php files.")
    os.chdir(str(CONTENT_FOLDER))
    run('rename *.php *.html')

@task
def copy_js():
    '''Copy Gigatree .js files.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Copy Gigatree .js files.")
    # files are copied from the base CONTENT_FOLDER (where they are put by unzipping
    # the adam.zip) to the CONTENT_FOLDER / js (where Pelican will find them)

    js_files = ('tab-list-handler.js',
                'tooltip-handler.js',
                'graph-handler.js',
                'gigatrees-map-min.js', )

    for my_file in js_files:
        try:
            winshell.delete_file(str(CONTENT_FOLDER / 'js' / my_file), no_confirm=True, allow_undo=False, silent=True)
        except:
            pass
        winshell.copy_file(str(CONTENT_FOLDER / my_file), str(CONTENT_FOLDER / 'js' / my_file), no_confirm=True)

# not needed; the elements of the 'gigatrees.css' needed have been folded
# directly into the theme LESS files
@task
def copy_css():
    '''Copy Gigatree .css files.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Copy Gigatree .css files.")
    os.chdir(str(CONTENT_FOLDER))

    css_files = ('gigatrees.css', )

    for my_file in css_files:
        try:
            winshell.delete_file("../css/" + my_file, no_confirm=True, allow_undo=False, silent=True)
        except:
            pass
        winshell.copy_file(my_file, "../css/" + my_file, no_confirm=True)

@task
def copy_img():
    '''Copy Gigatree image files.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Copy Gigatree image files.")
    # files are copied from the base CONTENT_FOLDER (where they are put by unzipping
    # the adam.zip) to the CONTENT_FOLDER / img (where Pelican will find them)

    img_files = ('arrowd.png',
                 'arrowl.png',
                 'arrowr.png',
                 'arrowu.png',
                 'bg-black.png',
                 'bg-pattern.png',
                 'mapicon_f.png',
                 'mapicon_m.png',
                 'mapicon_u.png',
                 'mapmarker1.png',
                 'mapmarker2.png',
                 'mapmarker3.png',
                 'mapmarker4.png',
                 'mapmarker5.png',
                 'avatar.jpg',
                 'image.jpg',
                 'pdf.jpg', )

    for my_file in img_files:
        try:
            winshell.delete_file(str(CONTENT_FOLDER / 'img' / my_file), no_confirm=True, allow_undo=False, silent=True)
        except:
            pass
        winshell.copy_file(str(CONTENT_FOLDER / my_file), str(CONTENT_FOLDER / 'img' / my_file), no_confirm=True)

@task
def replace_index():
    '''Copy over index.md, 404.md.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Copy over index.md, 404.md.")

    os.chdir(str(CONTENT_FOLDER))
    for my_file in ("index.md", "index.html", "404.md"):
        try:
            winshell.delete_file(my_file, no_confirm=True, allow_undo=False, silent=True)
        except:
            pass

    for my_file in ("index.md", "404.md"):
        winshell.copy_file("../../_unchanging_pages/{}".format(my_file), my_file, no_confirm=True)

@task
def set_pelican_variables():
    '''Sets a couple of variables that Pelican uses while generating the site.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Setting up Pelican.")

    adam_version_text = get_adam_version()  # 'Built by Adam 1.35.0.0 ' or the like
    adam_version_text = adam_version_text.decode('utf-8').strip()
    date_in_text = date.today().strftime("%B %d, %Y").replace(' 0', ' ')  # 'January 7, 2014' or the like
    year_range = "{}-{}".format(COPYRIGHT_START_YEAR, datetime.now().year)
    print('{}{} - {}'.format(INDENT, adam_version_text, date_in_text))

    f = open(str(WORKING_FOLDER / 'adamconf.py'), 'w')
    f.write('# Genealogy Uploader, v.{}\n'.format(str(__version__)))
    f.write('# {}\n\n'.format(str(MY_GEDCOM)))
    f.write('ADAM = True\n')
    f.write('ADAM_VERSION = "{}"\n'.format(adam_version_text))
    f.write('ADAM_UPDATED = "{}"\n'.format(date_in_text))
    f.write('ADAM_COPY_DATE = "{}"\n'.format(year_range))
    f.write('ADAM_LINK = "{}"\n'.format(ADAM_LINK))
    f.write('ADAM_FOOTER = "{}"\n'.format(ADAM_FOOTER))
    f.write('ADAM_PUBLISH = True\n')
    f.close()

@task
def replace_emails():
    '''Hide emails in Sources.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Hiding Emails.")

    # replace and hide emails; but some of these are over lines breaks,
    #  so we'll have to search and replace through the output
    # We are actually only considering 'Sources', as that's where
    #  all the emails seem to be...
    replacements =  ("someone@hotmail.com",           '[email redacted]'), \
                    ("someone@gmail.com",             '[email redacted]'),

    os.chdir(str(CONTENT_FOLDER))
    all_files = os.listdir(str(CONTENT_FOLDER))
    all_html_files = []
    for my_file in all_files:
        #if my_file.endswith(".html"):
        if my_file.startswith('sources-'):
            all_html_files.append(my_file)
    counter = 0
    bar = minchin.text.progressbar(maximum=len(all_html_files))
    # inline search and replace
    for my_file in all_html_files:
        for line in fileinput.input(my_file, inplace=1):
            print(multiple_replace(line, *replacements))
        counter += 1
        bar.update(counter)
    print()  # clear progress bar

def html_fixes(my_file):
    '''
    Applies the various updates and changes I want done to the raw Gigatrees
    HTML.

    Assumes 'my_file' is in the CONTENT_FOLDER.
    Assumes 'my_file' is a string.
    '''

    with codecs.open(str(CONTENT_FOLDER / my_file), 'r', 'utf-8') as html_doc:
        my_html = html_doc.read()

    soup = BeautifulSoup(my_html, "lxml")

    # change page title
    title_tag = soup.html.head.title
    for tag in soup(id="gt-page-title", limit=1):
        title_tag.string.replace_with(tag.string.strip())
        tag.decompose()

    # dump all the meta tags in the head section
    for tag in soup("meta"):
        tag.decompose()

    # Remove links to CDN stuff I serve locally
    js_served_locally = ('jquery.min.js',
                         'jquery-ui.min.js',
                         'bootstrap.min.js',
                         'globalize.min.js',
                         'dx.chartjs.js')
    for tag in soup("script"):
        try:
            link = tag["src"]
            if link.endswith(js_served_locally):
                tag.decompose()
        except:
            pass

    # fix pdf paths?

    # other stuff
    for tag in soup(id="gt-page-title"):
        tag.decompose()
    for tag in soup(class_="gt-version", limit=1):
        tag.decompose()

    # at meta tags, used for the breadcrumbs in the link.
    # they need to be in the <head> section, in the form
    #
    # <head>
    #    <!-- other stuff... -->
    #    <meta name="at" content="Locations" />
    #    <meta name="at_link" content="places.html" />  <!-- this is added to SITEURL -->
    # </head>
    new_tags = False
    if my_file.startswith("names-"):
        new_tag_1 = soup.new_tag("meta", content="Surnames")
        new_tag_2 = soup.new_tag("meta", content="names.html")
        new_tags = True
    elif my_file.startswith("places-"):
        new_tag_1 = soup.new_tag("meta", content="Locations")
        new_tag_2 = soup.new_tag("meta", content="places.html")
        new_tags = True
    elif my_file.startswith("sources-"):
        new_tag_1 = soup.new_tag("meta", content="Sources")
        new_tag_2 = soup.new_tag("meta", content="sources.html")
        new_tags = True
    elif my_file.startswith("timeline") and my_file != "timeline.html":
        new_tag_1 = soup.new_tag("meta", content="Timelines")
        new_tag_2 = soup.new_tag("meta", content="timeline.html")
        new_tags = True

    if new_tags:
        new_tag_1.attrs['name'] = 'at'
        new_tag_2.attrs['name'] = 'at_link'
        soup.html.head.append(new_tag_1)
        soup.html.head.append(new_tag_2)

    # write fixed version of file to disk
    with codecs.open(str(CONTENT_FOLDER / my_file), 'w', 'utf-8') as html_doc:
        html_doc.write(str(soup))
        #html_doc.write(soup.prettify())

def html_fixes_2(my_file, my_bar, my_counter):
    '''
    Applies the HTML changes and then updates the counter.

    Assumes 'my_bar' is of type minchin.text.progressbar
    Assumes 'my_counter' is of type multiprocessing.Value
    '''
    html_fixes(my_file)
    # get the lock so that different threads can't update it at the same time
    with my_counter.get_lock():
        my_counter.value += 1
        my_bar.update(my_counter.value)

'''
For 11,146 files (2 core, 4 thread machine, program run time):
    - no html cleaning (so just overhead):
        13:50 (partial delete, 7-Zip), 24:52, 21:56, 25:59 (Zipfile)
    - single thread: 18:49.190113
    - multi-threaded (n_jobs=9): 4:59.657586
'''

@task
def clean_adam_html_single_thread():
    '''Remove nasty and extra HTML.'''
    global step_no
    start_time_local = datetime.now()
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Remove nasty and extra HTML (single threaded).")

    #os.chdir(str(CONTENT_FOLDER))
    all_files = os.listdir(str(CONTENT_FOLDER))
    all_html_files = []
    for my_file in all_files:
        if my_file.endswith(".html"):
            all_html_files.append(my_file)

    # this gives us a C-type integer, useful for accessing from multiple threads
    counter = mp.Value('i', 0)
    bar = minchin.text.progressbar(maximum=len(all_html_files))
    bar.update(counter.value)
    for my_file in all_html_files:
        html_fixes_2(my_file, bar, counter)
    print()  # clear progress bar
    print(INDENT, datetime.now() - start_time_local)

@task
def clean_adam_html_multithreaded(my_n_jobs=None):
    '''Remove nasty and extra HTML (multi-threaded).'''
    '''
    Benchmarks

    n_jobs      time
    9           6:20
    6           6:18  (machine has 6 cores)
    4           7:16
    '''
    global step_no
    #start_time_local = datetime.now()
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Remove nasty and extra HTML (multi-threaded).")

    all_files = os.listdir(str(CONTENT_FOLDER))
    all_html_files = []
    for my_file in all_files:
        if my_file.endswith(".html"):
            all_html_files.append(my_file)

    if my_n_jobs is None:
        my_n_jobs = int(os.cpu_count())
    Parallel(n_jobs=my_n_jobs, verbose=5)(delayed(html_fixes)(my_file) for my_file in all_html_files)
    #print(INDENT, datetime.now() - start_time_local)

@task
def pelican():
    '''Run Pelican.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Run Pelican (site generator).")

    os.chdir(str(WORKING_FOLDER))
    run('pelican -s publishconf.py')

@task
def pelican_local():
    '''Run Pelican (in local, developmental mode).'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Run Pelican (site generator) in local mode.")

    os.chdir(str(WORKING_FOLDER))
    run('pelican -s pelicanconf.py')

@task
def create_tracking():
    '''Create deploy tracking file.'''
    global step_no
    global tracking_filename
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Create deploy tracking file.")

    # create a 'random' number using UUID
    # note that the last set of digits will correspond to the workstation
    myUUID = str(uuid.uuid1())
    tracking_filename = myUUID + ".txt"
    target = open(str(GITHUB_FOLDER / tracking_filename), 'w')
    target.write(myUUID + "\n")
    target.write("Adam upload by Python script.\n")
    target.write(GEDCOM_EXPECTED + "\n")
    target.close()

@task
def git():
    '''Git commit and push.'''
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Git -> commit and push.")

    commit_msg = "Gigatrees generated upload from {}".format(GEDCOM_EXPECTED)
    os.chdir(str(GITHUB_FOLDER))
    minchin.text.clock_on_right('{}{}> git add -A{}'.format(INDENT, Fore.YELLOW, Style.RESET_ALL))
    r1 = run('git add -A', hide=True)
    print(r1.stderr)
    minchin.text.clock_on_right('{}{}> git commit -m "{}"{}'.format(INDENT, Fore.YELLOW, commit_msg, Style.RESET_ALL))
    r2 = run('git commit -m Gigatrees_upload', hide=True)
    print(r2.stderr)
    minchin.text.clock_on_right('{}{}> git push origin{}'.format(INDENT, Fore.YELLOW, Style.RESET_ALL))
    r3 = run('git push origin', hide=True)
    print(r3.stdout)
    print(r3.stderr)

@task
def live():
    '''Tell us when we're live.'''
    # TO-DO: find tracking file based on creation/modified date
    global step_no
    step_no += 1
    minchin.text.clock_on_right(str(step_no).rjust(2) + ". Wait to go live.")

    if tracking_filename is None:
        print('{}No tracking file set.'.format(INDENT))
    else:
        while True:
            r = requests.head(URL_ROOT + "/" + tracking_filename, allow_redirects=True)
            if r.status_code == requests.codes.ok:
                break
            else:
                minchin.text.wait(180)

@task
def all_steps():
    '''Everything!'''
    minchin.text.title("Genealogy Uploader, v.{}".format(str(__version__)))
    print()

    export_gedcom()             # works 151230
    clean_gedcom()              # works
    upload_gedcom()             # works
    #check_images()              # works
    delete_old_output()         # works
    delete_old_adam()           # works ~2 min
    get_new_adam()              #
    unzip_adam()                # pretty sure works ~5 min
    #php_to_html()               # works, brakes if there are no PHP files
    copy_js()                   # works
    #copy_css()
    copy_img()
    replace_index()             # works
    set_pelican_variables()     # works
    clean_adam_html_multithreaded()
    replace_emails()            # doesn't crash
    create_tracking()           # works ~10 sec
    pelican()                   # works (assuming Pelican works)
    git()                       #
    live()                      #

    minchin.text.clock_on_right(Fore.GREEN + Style.BRIGHT + "Update is Live!")
    print(Style.RESET_ALL)

    print(INDENT, datetime.now() - start_time)

@task(default=True)
def does_nothing():
    print('this does nothing')

if __name__ == "__main__":
    all_steps()

#!/usr/bin/env python
# -*- coding: utf-8 -*- #

import os
import sys
sys.path.append(os.curdir)

# Adam configuration options
ADAM_PUBLISH = False
from adamconf import *
# ADAM = True  is used by the theme tempates to display 'Genealogy only' things

AUTHOR = 'D. Minchin & Wm. Minchin'
SITENAME = 'Minchin.ca'
if ADAM_PUBLISH:
    SITEURL = 'http://minchin.ca/genealogy'
    RELATIVE_URLS = False
else:
    SITEURL = '.'
    # Uncomment following line if you want document-relative URLs when developing
    RELATIVE_URLS = True
SITE_ROOT_URL = 'http://minchin.ca'

TIMEZONE = 'America/Edmonton'

DEFAULT_LANG = 'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None

DEFAULT_PAGINATION = False

# static paths will be copied under the same name
# these are relative to the base CONTENT folder
STATIC_PATHS = ['images',
                '../extras',
                'css',
                'design',
                'js',
                'pages/img',
                '../.gitattributes',
                '../.gitignore',
                '../README.txt',
                ]

# A list of files to copy from the source to the destination
EXTRA_PATH_METADATA = {
    '../.gitattributes':            {'path': '.gitattributes'},
    '../.gitignore':                {'path': '.gitignore'},
    '../README.txt':                {'path': 'README.txt'},
    '../extras/minchin.ico':        {'path': 'favicon.ico'},
    '../extras/.nojekyll':          {'path': '.nojekyll'},
    'js/tab-list-handler.js':       {'path': 'tab-list-handler.js'},
    'js/tooltip-handler.js':        {'path': 'tooltip-handler.js'},
    'js/graph-handler.js':          {'path': 'graph-handler.js'},
    'js/gigatrees-map-min.js':      {'path': 'gigatrees-map-min.js'},
    'pages/img/arrowd.png':         {'path': 'arrowd.png'},
    'pages/img/arrowl.png':         {'path': 'arrowl.png'},
    'pages/img/arrowr.png':         {'path': 'arrowr.png'},
    'pages/img/arrowu.png':         {'path': 'arrowu.png'},
    'pages/img/bg-black.png':       {'path': 'bg-black.png'},
    'pages/img/bg-pattern.png':     {'path': 'bg-pattern.png'},
    'pages/img/mapicon_f.png':      {'path': 'mapicon_f.png'},
    'pages/img/mapicon_m.png':      {'path': 'mapicon_m.png'},
    'pages/img/mapicon_u.png':      {'path': 'mapicon_u.png'},
    'pages/img/mapmarker1.png':     {'path': 'mapmarker1.png'},
    'pages/img/mapmarker2.png':     {'path': 'mapmarker2.png'},
    'pages/img/mapmarker3.png':     {'path': 'mapmarker3.png'},
    'pages/img/mapmarker4.png':     {'path': 'mapmarker4.png'},
    'pages/img/mapmarker5.png':     {'path': 'mapmarker5.png'},
    'pages/img/avatar.jpg':         {'path': 'avatar.jpg'},
    'pages/img/image.jpg':          {'path': 'image.jpg'},
    'pages/img/pdf.jpg':            {'path': 'pdf.jpg'},
    }

# Custom settings
FILENAME_METADATA = '(?P<slug>[\w-]*)'      # so anything before the file extension becomes the slug
## Please note that the metadata available inside your files takes precedence
#  over the metadata extracted from the filename.

MARKUP = (('rst',
           'md',
           'markdown',
           'mkd',
           'mdown',
           'html',
           'htm'))
PATH = 'content'
OUTPUT_PATH = '../genealogy-gh-pages/'

# Add Blog to sidebar
MENUITEMS = (('Blog',        'http://blog.minchin.ca/',      'fa fa-pencil'),
             ('Genealogy',   SITEURL,                        'glyphicon glyphicon-tree-deciduous'),
             ('My Projects', 'http://minchin.ca/projects/',  'fa fa-flask'),
             ('Search',      'http://minchin.ca/search/',    'fa fa-search'),
             ('About',       'http://minchin.ca/about/',     'fa fa-info-circle'),
             ('Contact Me',  'http://minchin.ca/contact/',   'fa fa-envelope'),
             )

MENUITEMS_2_AT = 'Genealogy'
MENUITEMS_2_AT_LINK = ''  # this is added to SITEURL

MENUITEMS_2 = (('Surnames',         SITEURL + '/names.html',          False),
               ('Updates',          SITEURL + '/updates.html',        False),
               ('Sources',          SITEURL + '/sources.html',        False),
               ('Distribution Map', SITEURL + '/map.html',            False),
               ('Timelines',        SITEURL + '/timeline.html',       False),
               #('Immigrants',       SITEURL + '/immigrants.html',     False),  # doens't exist in current builds
               #('Nobility',         SITEURL + '/titles.html',         False),  # doens't exist in current builds
               ('Locations',        SITEURL + '/places.html',         False),
               ('Bonkers Report',   SITEURL + '/bonkers-report.html', False),
               ('Photos',           SITEURL + '/photos.html',         False),
               #('External Links',   SITEURL + '/links.html',          False),  # doens't exist in current builds
               #('Statistics',       SITEURL + '/stats.html',          False),  # stats graphs aren't working right now
               )

DISPLAY_PAGES_ON_MENU = False

# disable Tags, etc
TAGS_SAVE_AS = ''
TAG_SAVE_AS = ''
CATEGORY_URL = ''
CATEGORY_SAVE_AS = ''
CATEGORIES_URL = ''
CATEGORIES_SAVE_AS = ''
ARTICLE_URL = ''
ARTICLE_SAVE_AS = ''
AUTHORS_URL = ''
AUTHORS_SAVE_AS = ''
ARCHIVES_URL = ''
ARCHIVES_SAVE_AS = ''
PAGE_URL = "{slug}.html"
PAGE_SAVE_AS = "{slug}.html"

# Theme Related
TYPOGRIFY = False  # turn off for HIDDEN names...
THEME = '../minchinweb.github.io-pelican/themes/pelican-minchin-ca'
SITELOGO = 'images/MinchindotCA-200.png'
SITELOGO_SIZE = '100%'
PYGMENTS_STYLE = 'friendly'
DISPLAY_BREADCRUMBS = True
FAVICON = 'favicon.ico'
BOOTSTRAP_THEME = 'minchin-ca'
USE_OPEN_GRAPH = True
CUSTOM_CSS = 'css/minchin-ca.css'
DOCUTIL_CSS = False
CUSTOM_JS_LIST = ['js/jquery-ui.min.js',
                  'js/globalize.min.js',
                  'js/dx.chartjs.js',
                  ]

GOOGLE_ANALYTICS_UNIVERSAL = 'UA-384291-3'
GOOGLE_ANALYTICS_UNIVERSAL_PROPERTY = 'minchin.ca'

# Plugins
PLUGIN_PATHS = ('../pelican-plugins',)
PLUGINS = ['assets', ]

ASSET_CSS = False
ASSET_JS = False

SITEMAP = {
    "format": "xml",
}

# # Make things disappear
DISPLAY_CATEGORIES_ON_MENU = False
HIDE_SITENAME = True
HIDE_SIDEBAR = True
FEED_ALL_ATOM = False
FEED_ALL_RSS = False
GITHUB_USER = False
ADDTHIS_PROFILE = False
DISQUS_SITENAME = False
PDF_PROCESSOR = False

# Genealogy Uploader, v.3.2.3
# C:\Users\William\Desktop\William 160108.ged

ADAM = True
ADAM_VERSION = "Built by Gigatrees (3.0.14)"
ADAM_UPDATED = "January 8, 2016"
ADAM_COPY_DATE = "1987-2016"
ADAM_LINK = "http://gigatrees.com"
ADAM_FOOTER = "<p><strong>Are we related?</strong> Are you a long lost cousin? Spotted an error here? This website remains a work-in-progress and I would love to hear from you. Drop me a line at minchinweb [at] gmail [dot] com.</p>"
ADAM_PUBLISH = True

package
  __init__.py
  _compiled_2_7
    package
      __init__.py
      main.py
  _compiled_3_3
    package
      __init__.py
      main.py
  _compiled_3_4
    package
      __init__.py
      main.py
  _compiled_3_5
    package
      __init__.py
      main.py
  _compiled_3_6
    package
      __init__.py
      main.py

# __init__.py
print('init')

# main.py
print('main')

# REPL
>>> import package.main
init
main

>>> from package import main
init
main

# shell
➜ python -m package.main
init
main

import sys
import os

VERSIONS = {'_compiled_2_7': (2, 7),
            '_compiled_3_0': (3, 0),
            '_compiled_3_1': (3, 1),
            '_compiled_3_2': (3, 2),
            '_compiled_3_3': (3, 3),
            '_compiled_3_4': (3, 4),
            '_compiled_3_5': (3, 5),
            '_compiled_3_6': (3, 6)}
root = os.path.abspath(os.path.dirname(__file__))

def _get_available_versions():
    for version in os.listdir(root):
        if version in VERSIONS:
            yield version

def _get_version():
    available_versions = sorted(_get_available_versions())[::-1]
    for version in available_versions:
        if VERSIONS[version] <= sys.version_info:
            return version

# We should pass `__name__` as an argument, because
# we can't access `__name__` after module deletion
def _import_module(name):
    version = _get_version()
    version_path = os.path.join(root, version)
    sys.path.insert(0, version_path)
    del sys.modules[name]
    __import__(name)

_import_module(__name__)

print('init 2.7')  # replace with required version

print('main 2.7')  # replace with required version

# REPL
>>> import package
init 2.7

>>> import package.main
init 2.7
main 2.7

>>> from package import main
init 2.7
main 2.7

# shell
➜ python2 -m package.main
init 2.7
main 2.7

# REPL
>>> import package
init 3.5

>>> import package.main
init 3.5
main 3.5

>>> from package import main
init 3.5
main 3.5

# shell
➜ python3 -m package.main
init 3.5
main 3.5

Planet Python

BangPypers: How BangPypers meetup is run

What is BangPypers?

Event Types?

What are the task involved?

Venue

Speakers

RSVP

Managing meetup page

On the day

Blog

Feedback form

Planning

Video

Programming Ideas With Jake: Another Look at Instance-Level Properties in Python

Sandipan Dey: Some NLP: Probabilistic Context Free Grammar (PCFG) and CKY Parsing in Python

François Dion: Raspberry Pi 3 Canakit

It's been a long time...

Raspberry Pi 3

The Canakit

No SD card?

Python Data: Visualizing data – overlaying charts in python

An example

Frank Wierzbicki: Jython 2.7.1 release candidate 2 released!

PyBites: Flask Sessions

Import Python: ImportPython Issue 127 - Python Functions aren't what you think, API checklist and more

Python Bytes: #28 The meaning of _ in Python

Kushal Das: My lightning talk in Django Girls PyCon

Django Weblog: Django bugfix release: 1.11.2

Will McGugan: Announcing Lomond, a WebSocket Client Library for Python

Why "Lomond"?

Design

Roadmap

More Information

Ian Ozsvald: PyDataLondon 2017 Conference write-up

Experienced Django: Playing with Generators

Simple Generator Example

Better Generator Solution

map

while any

min enumerate

the rest

Heapq

Thanks

PyCharm: PyCharm Edu 4 Early Access Program is Open

What is EAP and why should you join?

What’s new in PyCharm Edu 4

New Welcoming UI for Students

New Welcoming UI for Teachers

Wesley Chun: Managing Team Drives with Python and the Google Drive API

Introduction

Using the Google Drive API

Create Team Drives

Add members to Team Drives

Create a folder in Team Drives

Import/upload files to Team Drives folders

Driver app

Conclusion

Code challenge

Python Data: Forecasting Time-Series data with Prophet – Part 1

Installing Prophet

Getting started

PyCharm: PyCharm 2017.2 EAP 2 (build 172.2656.2)

Glyph Lefkowitz: The Sororicide Antipattern

William Minchin: Advanced Pelican: Automated Site Updates with Travis-CI and GitHub Pages

Static vs Dynamic Websites

GitHub for Storage

Pelican Extras on PyPI

Travis-CI for Processing

GitHub Pages for Site Hosting

Closing Thoughts

Daniel Bader: An Overview of Python’s “ipaddress” Module

An Overview of Python’s “ipaddress” Module

IPv4 vs IPv6 Addresses – A Primer

Python’s ipaddress Module and IPv4 Addresses

Python’s ipaddress Module and IPv6 Addresses

Additional Resources

William Minchin: Introduction to Pelican

Intended Audience

HTML Was Simple

Python’s `ipaddress` Module and IPv4 Addresses

Python’s `ipaddress` Module and IPv6 Addresses

Pelican Configuration — `pelicanconf.py`

`tasks.py` — Invoke configuration file

`floating_rate_vs_rro_graph.py` — to generate graphs

`pelicanconf.py` — Pelican Configuration

`pelicanconf.py` — Pelican Configuration

`gen_upload.py` — Build Automation Script

`pelicanconf.py` — Pelican Configuration

`adamconf.py` — Additional Settings File