[PYTHON] Forex automatic trading with genetic algorithm Part 3 Actual trading with Oanda API

Previous article

FX automatic trading system made with python and genetic algorithm part 1 Forex automatic trading with genetic algorithm Part 2 Evolving trading AI implementation

Things to make this time

I would like to write about the Oanda API that I used when I made the automatic trading system and the function of placing buy and sell orders.

Oanda API used for automatic trading

It was built using 5 types of APIs. For details, refer to Oanda's API Document OAuth2 Token can be issued on the Web by opening an account and logging in. Set the token to Authorization: Bearer ******** on the curl command Header.

"""
01.Account information API
curl -H "Authorization: Bearer ********" https://api-
fxtrade.oanda.com/v1/accounts/12345
"""
{
    "accountId" : 12345,
    "accountName" : "Primary",
    "balance" : 123456.5765,
    "unrealizedPl" : 36.8816,
    "realizedPl" : 235.5839,
    "marginUsed" : 123456.154,
    "marginAvail" : 123456.3041,
    "openTrades" : 20,
    "openOrders" : 0,
    "marginRate" : 0.04,
    "accountCurrency" : "JPY"
}


"""
02.Ordering API
curl -H "Authorization: Bearer ********"　-X POST -d "instrument=EUR_USD&units=2&side=sell&type=market" "https://api-fxtrade.oanda.com/v1/accounts/12345/orders"
"""
{

  "instrument" : "EUR_USD",
  "time" : "2013-12-06T20:36:06Z", // Time that order was executed
  "price" : 1.37041,               // Trigger price of the order
  "tradeOpened" : {
    "id" : 175517237,              // Order id
    "units" : 1000,                // Number of units
    "side" : "buy",                // Direction of the order
    "takeProfit" : 0,              // The take-profit associated with the Order, if any
    "stopLoss" : 0,                // The stop-loss associated with the Order, if any
    "trailingStop" : 0             // The trailing stop associated with the rrder, if any
  },
  "tradesClosed" : [],
  "tradeReduced" : {}
}

"""
03.Get the current position list
curl -H "Authorization: Bearer ####" https://api-fxtrade.oanda.com/v1/accounts/######/positions
"""
{
    "positions" : [
        {
            "instrument" : "AUD_USD",
            "units" : 100,
            "side" : "sell",
            "avgPrice" : 0.78216
        },
        {
            "instrument" : "GBP_USD",
            "units" : 100,
            "side" : "sell",
            "avgPrice" : 1.49128
        },
        {
            "instrument" : "USD_JPY",
            "units" : 850,
            "side" : "buy",
            "avgPrice" : 119.221
        }
}

"""
04.Get market rate
curl -H "Authorization: Bearer ********" -X GET "https://api-fxtrade.oanda.com/v1/prices?instruments=EUR_USD%2CUSD_JPY%2CEUR_CAD"
"""
{
    "prices" : [
        {
            "instrument" : "EUR_USD",
            "time" : "2015-03-13T20:59:58.165668Z",
            "bid" : 1.04927,
            "ask" : 1.04993,
            "status" : "halted"
        },
        {
            "instrument" : "USD_JPY",
            "time" : "2015-03-13T20:59:58.167818Z",
            "bid" : 121.336,
            "ask" : 121.433,
            "status" : "halted"
        },
        {
            "instrument" : "EUR_CAD",
            "time" : "2015-03-13T20:59:58.165465Z",
            "bid" : 1.34099,
            "ask" : 1.34225,
            "status" : "halted"
        }
    ]
}

"""
05.API to check the order status of buy and sell orders
curl -H "Authorization: Bearer ************" https://api-fxtrade.oanda.com/v1/accounts/****/transactions
"""
{
     "id" : 1789536248,
     "accountId" : *****,
     "time" : "2015-03-20T12:17:06.000000Z",
     "type" : "TAKE_PROFIT_FILLED",
     "tradeId" : 1789531428,
     "instrument" : "USD_JPY",
     "units" : 1,
     "side" : "sell",
     "price" : 121.017,
     "pl" : 0.02,
     "interest" : 0,
     "accountBalance" : 33299999.1173
},

          {
     "id" : 1789560748,
     "accountId" : *****,
     "time" : "2015-03-20T12:49:38.000000Z",
     "type" : "STOP_LOSS_FILLED",
     "tradeId" : 1789500620,
     "instrument" : "USD_JPY",
     "units" : 100,
     "side" : "sell",
     "price" : 120.899,
     "pl" : -18.6,
     "interest" : 0,
     "accountBalance" : 33299980.3023
}

Sample code to access the API

This is an implementation example of the price acquisition API. Web access with urllib request. The response will be returned in Json, so I'll do my best to write the parser class. When the spread opens (the market becomes rough and the difference between the bid price and the sell price opens), I tried to incorporate a function that does not place an order when the rate is unfavorable.

■ Sample code accessing the sandbox API http://api-sandbox.oanda.com/v1/prices?instruments=EUR_USD%2CUSD_JPY%2CGBP_USD%2CAUD_USD

price_api.py

# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import unicode_literals
import urllib
import requests
import time
import ujson
import pytz
from enum import Enum


class CurrencyPair(Enum):
    EUR_USD = 1
    USD_JPY = 2
    GBP_USD = 3
    AUD_USD = 4


class OandaAPIMode(Enum):
    PRODUCTION = 1
    DEVELOP = 2
    SANDBOX = 3
    DUMMY = 4

    @property
    def url_base(self):
        api_base_url_dict = {
            OandaAPIMode.PRODUCTION: 'https://api-fxtrade.oanda.com/',
            OandaAPIMode.DEVELOP: 'https://api-fxpractice.oanda.com/',
            OandaAPIMode.SANDBOX: 'http://api-sandbox.oanda.com/',
        }
        return api_base_url_dict.get(self)

    @property
    def headers(self):
        """
        :rtype : dict
        """
        _base = {
            'Accept-Encoding': 'identity, deflate, compress, gzip',
            'Accept': '*/*', 'User-Agent': 'python-requests/1.2.0',
            'Content-type': 'application/x-www-form-urlencoded',
        }

        if self == OandaAPIMode.SANDBOX:
            return _base
        if self == OandaAPIMode.DEVELOP:
            _base['Authorization'] = 'Bearer {}'.format(get_password('OandaRestAPITokenDemo'))
            return _base
        if self == OandaAPIMode.PRODUCTION:
            _base['Authorization'] = 'Bearer {}'.format(get_password('OandaRestAPIToken'))
            return _base
        raise ValueError


class OandaServiceUnavailableError(Exception):
    """
Error during service stop
    """
    pass


class OandaInternalServerError(Exception):
    """
Error during service stop
    """
    pass


class OandaAPIBase(object):
    mode = None

    class Meta(object):
        abstract = True

    def __init__(self, mode):
        """
        :param mode: OandaAPIMode
        """
        self.mode = mode

    def requests_api(self, url, payload=None):
        #If communication fails, retry up to 3 times
        for x in xrange(3):
            response = None
            try:
                if payload:
                    response = requests.post(url, headers=self.mode.headers, data=payload)
                else:
                    response = requests.get(url, headers=self.mode.headers)

                assert response.status_code == 200, response.status_code
                print 'API_Access: {}'.format(url)
                data = ujson.loads(response.text)
                self.check_json(data)
                return data
            except Exception as e:
                if response is None:
                    raise
                if response.text:
                    if str("Service Unavailable") in str(response.text):
                        raise OandaServiceUnavailableError
                    if str("An internal server error occurred") in str(response.text):
                        raise OandaInternalServerError
                time.sleep(3)
                if x >= 2:
                    raise ValueError, response.text
        raise

    def check_json(self, data):
        raise NotImplementedError


class PriceAPI(OandaAPIBase):
    """
Rate confirmation API

    %2C is a comma

    curl -H "Authorization: Bearer ********" -X GET "https://api-fxtrade.oanda.com/v1/prices?instruments=EUR_USD%2CUSD_JPY%2CEUR_CAD"

    {
        "prices" : [
            {
                "instrument" : "EUR_USD",
                "time" : "2015-03-13T20:59:58.165668Z",
                "bid" : 1.04927,
                "ask" : 1.04993,
                "status" : "halted"
            },
            {
                "instrument" : "USD_JPY",
                "time" : "2015-03-13T20:59:58.167818Z",
                "bid" : 121.336,
                "ask" : 121.433,
                "status" : "halted"
            },
            {
                "instrument" : "EUR_CAD",
                "time" : "2015-03-13T20:59:58.165465Z",
                "bid" : 1.34099,
                "ask" : 1.34225,
                "status" : "halted"
            }
        ]
    }

    :rtype : dict of PriceAPIModel
    """
    url_base = '{}v1/prices?'

    def get_all(self):
        instruments = ','.join([x.name for x in CurrencyPair])
        instruments = urllib.quote(instruments, '')
        url = self.url_base.format(self.mode.url_base)
        url += 'instruments={}'.format(instruments)
        data = self.requests_api(url)
        d = {}
        for price in data.get('prices'):
            price_model = PriceAPIModel(price)
            d[price_model.currency_pair] = price_model
        return d

    def check_json(self, data):
        assert 'prices' in data


class PriceAPIModel(object):
    """
Class that parses the rate confirmation API
    """
    instrument = None
    time = None
    bid = None  #Tick at the time of SELL
    ask = None  #Tick at the time of BUY
    status = None

    def __init__(self, price):
        self.ask = float(price.get('ask'))
        self.bid = float(price.get('bid'))
        self.time = parse_time(price.get('time'))
        self.instrument = str(price.get('instrument'))
        if 'status' in price:
            self.status = str(price.get('status'))
        self._check(price)

    def _check(self, price):
        if 'ask' not in price:
            raise ValueError
        if 'bid' not in price:
            raise ValueError
        if 'time' not in price:
            raise ValueError
        if 'instrument' not in price:
            raise ValueError
        self.currency_pair

    @property
    def currency_pair(self):
        """
        :rtype : CurrencyPair
        """
        for pair in CurrencyPair:
            if str(pair.name) == str(self.instrument):
                return pair
        raise ValueError

    @property
    def is_maintenance(self):
        """
True if during maintenance
        :rtype : bool
        """
        if self.status is None:
            return False
        if str(self.status) == str('halted'):
            return True
        return False

    def is_active(self):
        """
True if valid rate
        :rtype : bool
        """
        #Not in maintenance
        if self.is_maintenance:
            return False

        #Allows up to 4 ticks normal rate
        if self.cost_tick > 4:
            return False
        return True


def get_password(key):
    """
Respond to password
    """
    return "12345"


def parse_time(time_text):
    """
Convert a string to UTC.
Example)
    2015-02-22T15:00:00.000000Z
    :param time_text: char
    :rtype : datetime
    """
    import datetime
    t = time_text
    utc = datetime.datetime(int(t[0:4]),
                            int(t[5:7]),
                            int(t[8:10]),
                            int(t[11:13]),
                            int(t[14:16]),
                            int(t[17:19]),
                            int(t[20:26]), tzinfo=pytz.utc)
    return utc


def utc_to_jst(utc):
    return utc.astimezone(pytz.timezone('Asia/Tokyo'))


print "START"
# API_Access
price_group_dict = PriceAPI(OandaAPIMode.SANDBOX).get_all()

#Print the result
for key in price_group_dict:
    pair = key
    price = price_group_dict[key]
    print "Pair:{} bid:{} ask:{} time:{}".format(pair.name, price.bid, price.ask, price.time)
print "FINISH"

Execution result

>python price_api.py 
START
API_Access: http://api-sandbox.oanda.com/v1/prices?instruments=EUR_USD%2CUSD_JPY%2CGBP_USD%2CAUD_USD
Pair:USD_JPY bid:120.243 ask:120.293 time:2015-09-23 21:07:30.363800+00:00
Pair:AUD_USD bid:0.70049 ask:0.70066 time:2015-09-23 21:07:30.367431+00:00
Pair:EUR_USD bid:1.13916 ask:1.1393 time:2015-10-13 07:16:53.931425+00:00
Pair:GBP_USD bid:1.52455 ask:1.5248 time:2015-09-23 21:07:30.362069+00:00
FINISH

Unexpected errors often occur in production systems. Since the Oanda API is simple, I think it will be easier to write the library yourself later. If you want to use the library, it is better to use the library considering that the response of the Saturday and Sunday API will stop.

A nightmare when 100 AIs are bought and sold (double-decker problem)

When 100 AI buys and sells dollar yen at the timing when the 5-minute bar is updated, if buy is 60 and sell is 40, the overlapping 40 parts will be denominated in both and will be lost in the commission. As a countermeasure for the double-decker problem, we will optimize buying and selling in bulk orders.

■ Sample case Minimize trading fees when AI1-6 place an order as follows. It is assumed that one transaction fee is 100 yen.

■ Method 1. At 120.00 yen, place an order for 4 Buy and 2 Sell (trading fee 600 yen) After that, the market price fluctuates and all 6 orders are consumed (trading fee 600 yen). 1200 yen in total

■ Method 2. At the time of 120.00 yen, the orders of AI1 to 6 are put together and the dollar yen 120.00 Buy 2 orders (trading fee 200 yen) After that, the market price fluctuates and all 6 orders are consumed (trading fee 600 yen). 800 yen in total

Since Oanda is a double-decker system that does not have a position, I decided to buy and sell using the solution 2.

If you use the bulk order method for buying and selling orders, there will be a side effect that you will not be able to place Limit (profit) and Stop Limit (stop loss at limit) orders.

In the solution 1 method, you can set Limit and Stop Limit for each order and place an order, so once you place the order, Oanda will automatically settle the payment, but if you use the solution 2 method, the order unit will be different, so you can place an order. You also need to place an order for payment from your own program.

Introduce supervisord to increase availability

What if the payment program stops with an error? The system will continue to place orders, but the positions will not be closed. Leverage of your account will continue to rise. To prevent this from happening, trading programs are required to have high availability. I also installed Call notification function when an error occurs.

To achieve high availability, it is easy to process the program with supervisord. Create open and close commands for positions (buy and sell orders) in Python and use supervisord to launch the process. Similar functions can be achieved with Cron and while True, but they have many disadvantages and were not adopted.

order_open.py

# -*- coding: utf-8 -*-
#It's a pseudo-language because it's empowered. Maybe it doesn't work. sorry
from __future__ import absolute_import
from __future__ import unicode_literals
import datetime
from django.core.management import BaseCommand
import time
import sys
import pytz
from requests import ConnectionError

ACCOUNT_ID = 12345


class CustomBaseCommand(BaseCommand):
    class Meta(object):
        abstract = True

    def echo(self, txt):
        d = datetime.datetime.today()
        print d.strftime("%Y-%m-%d %H:%M:%S") + ':' + txt


class Command(CustomBaseCommand):
    """
Place an order using BoardAI
    """
    CACHE_PREV_RATES = {}

    def handle(self, *args, **options):
        try:
            self.run()
        except OandaServiceUnavailableError:
            #During maintenance on Saturdays and Sundays
            self.echo("ServiceUnavailableError")
            time.sleep(60)
        except OandaInternalServerError:
            #During maintenance on Saturdays and Sundays
            self.echo("OandaInternalServerError")
            time.sleep(60)
        except ConnectionError:
            time.sleep(60)

        #Stop for 3 seconds
        time.sleep(3)

    def run(self):
        #Margin check
        account = AccountAPI(OandaAPIMode.PRODUCTION, ACCOUNT_ID).get_all()
        if int(account.get('marginAvail')) < 10000:
            self.echo('MARGIN EMPTY!! marginAvail:{}'.format(int(account.get('marginAvail'))))
            time.sleep(3000)

        #take price
        price_group = PriceAPI(OandaAPIMode.PRODUCTION).get_all()

        #AI instance generation
        order_group = []
        ai_board_group = AIBoard.get_all()

        #Temporary order
        now = datetime.datetime.now(tz=pytz.utc)
        for ai_board in ai_board_group:
            order = self.pre_order(ai_board, price_group, now)
            if order:
                order_group.append(order)

        #Order
        self.order(order_group, price_group)

    def pre_order(self, ai_board, price_group, now):
        """
Return True after placing an order
        :param ai_board: AIBoard
        :param price_group: dict of PriceAPIModel
        :param now: datetime
        :rtype : bool
        """
        ai = ai_board.get_ai_instance()
        price = price_group.get(ai.currency_pair, None)

        #Is the price normal?
        if price is None:
            return None
        if not price.is_active():
            # print 'price not active'
            return None

        #Rate is not normal
        prev_rates = self.get_prev_rates(ai.currency_pair, Granularity.H1)
        if not prev_rates:
            return None
        prev_rate = prev_rates[-1]

        #Deviation from the previous rate within 3 minutes
        if now - prev_rate.start_at > datetime.timedelta(seconds=60 * 3):
            # print 'ORDER TIME IS OVER'
            return None

        #Purchase limit by number of positions and purchase limit by time
        if not ai_board.can_order(prev_rate):
            # print 'TIME OR POSITION LIMIT'
            return None

        #Purchase decision Similar to AI simulation, buy and sell is decided only for mid
        order_ai = ai.get_order_ai(prev_rates, price.mid, price.time, is_production=True)

        if order_ai is None:
            return None
        if order_ai.order_type == OrderType.WAIT:
            return None

        #Temporary order firing
        order = Order.pre_open(ai_board, order_ai, price, prev_rate.start_at)
        return order

    def order(self, order_group, price_group):
        """
Take a summary of the pre-order and actually place the order
        :param order_group: list of Order
        :param price_group: dict of PriceAPIModel
        :return:
        """
        #Take a summary
        order_dict = {x: 0 for x in CurrencyPair}
        for order in order_group:
            if order.buy:
                order_dict[order.currency_pair] += order.units
            else:
                order_dict[order.currency_pair] -= order.units
        print order_dict

        #Order
        api_response_dict = {}
        for key in order_dict:
            if order_dict[key] == 0:
                print '{}:NO ORDER'.format(key)
                continue
            print '{}:ORDER!'.format(key)
            units = order_dict[key]
            api_response_dict[key] = OrdersAPI(OandaAPIMode.PRODUCTION, ACCOUNT_ID).post(key, units, tag='open')

        #DB update
        for order in order_group:
            order.open(price_group.get(order.currency_pair))

    def get_prev_rates(self, currency_pair, granularity):
        key = self._get_key(currency_pair, granularity)
        print key
        r = self.CACHE_PREV_RATES.get(key)
        if r:
            return r
        prev_rates = CurrencyPairToTable.get_table(currency_pair, granularity).get_new_record_by_count(10000)
        self.CACHE_PREV_RATES[key] = prev_rates
        return prev_rates

    def _get_key(self, currency_pair, granularity):
        return 'RATE:{}:{}'.format(currency_pair.value, granularity.value)

order_close.py

# -*- coding: utf-8 -*-
#It's a pseudo-language because it's empowered. Maybe it doesn't work. sorry
from __future__ import absolute_import
from __future__ import unicode_literals
import datetime
from django.core.management import BaseCommand
import time
import pytz
from requests import ConnectionError

class Command(CustomBaseCommand):
    """
Close using account history
    """
    CACHE_PREV_RATES = {}

    def handle(self, *args, **options):
        print '********************************'
        self.echo('close start')
        self.check_kill_switch()

        try:
            self.run()
        except OandaServiceUnavailableError:
            #During maintenance on Saturdays and Sundays
            self.echo("ServiceUnavailableError")
            time.sleep(60)
        except OandaInternalServerError:
            #During maintenance on Saturdays and Sundays
            self.echo("OandaInternalServerError")
            time.sleep(60)
        except ConnectionError:
            time.sleep(60)
        except Exception as e:
            self.critical_error('close', e)
        time.sleep(3)

    def run(self):
        orders = Order.get_open()

        #take price
        price_group = PriceAPI(OandaAPIMode.PRODUCTION).get_all()

        #Take a summary
        order_dict = {x: 0 for x in CurrencyPair}
        for order in orders:
            if order.can_close(price_group[order.currency_pair]):
                if order.buy:
                    #Reverse buying and selling
                    print order.currency_pair, order.units
                    order_dict[order.currency_pair] -= order.units
                    print order_dict[order.currency_pair]
                else:
                    #Reverse buying and selling
                    print order.currency_pair, order.units
                    order_dict[order.currency_pair] += order.units
                    print order_dict[order.currency_pair]
        print order_dict

        #Order
        api_response_dict = {}
        for key in order_dict:
            if order_dict[key] == 0:
                print '{}:NO ORDER'.format(key)
                continue
            print '{}:ORDER'.format(key)
            units = order_dict[key]
            api_response_dict[key] = OrdersAPI(OandaAPIMode.PRODUCTION, 12345).post(key, units, tag='close')

        #Record
        for order in orders:
            if order.can_close(price_group[order.currency_pair]):
                order.close(price_group.get(order.currency_pair))

Build a computational cluster of genetic algorithms

I tried to build a calculation cluster of genetic algorithm with oleore specifications. The worker makes a command to calculate with a genetic algorithm in python and starts it with supervisord.

Spoilers: Transaction history

Was it profitable to buy and sell automatically after all?

Thank you for staying with us until the end. The development of the automatic trading system was very interesting. Please try it.

Related article

FX automatic trading system made with python and genetic algorithm part 1 Forex automatic trading with genetic algorithm Part 2 Evolving trading AI implementation Forex automatic trading with genetic algorithm Part 3 Actual trading with Oanda API