RMBS Prototype Model in Python

Parameters

Parameters including : Total balance , Attach point, mortgage rate , Tranche Rate, Payment term and Prepayment CPR .

newscreen140314116

 Cash Flow Chart result:

newscreen140405330

Full code(2 functions):

    def rmbs_show(self):
        self.clr_update_frame()

        Label( self.update_frame, text="CPR" ).grid( row = 0, column= 0 )    
        w = Scale( self.update_frame, from_=0.01, to=0.20, resolution = 0.005, orient=HORIZONTAL, variable=self.rmbs_cpr, showvalue = True )
        w.grid(row = 0, column = 1)

        cb_mortgage_type = Checkbutton(self.update_frame,variable=self.rmbs_mortgageType ,text="Fix Principal")
        cb_mortgage_type.grid(row=0, column=2)

        Label( self.update_frame, text="Outstanding Bal" ).grid( row = 1, column= 0 )
        e = Entry( self.update_frame)
        e.insert(0,"100000")
        e.grid(row = 2, column =0)

        Label( self.update_frame, text="Period").grid(row = 1, column=1 )
        term = Entry(self.update_frame)
        term.insert(0,"360")
        term.grid(row = 2, column=1)

        Label( self.update_frame, text="Mortgage Rate").grid(row = 1, column=2 )
        mortgage_rate = Entry(self.update_frame)
        mortgage_rate.insert(0,"0.16")
        mortgage_rate.grid(row = 2, column=2)

        Label( self.update_frame, text="Senior Tranche" ).grid( row = 4, column= 0 )
        de = Entry( self.update_frame)
        de.grid(row = 5, column =0)
        de.insert(0,"50000")

        Label( self.update_frame, text="Senior Rate" ).grid( row = 4, column= 1 )
        de_rate = Entry( self.update_frame )
        de_rate.grid(row = 5, column = 1)
        de_rate.insert(0,"0.03")

        Label( self.update_frame, text="Junior Tranche" ).grid( row = 6, column= 0 )
        de1 = Entry( self.update_frame)
        de1.grid(row = 7, column =0)
        de1.insert(0,"10000")

        Label( self.update_frame, text="Junior Rate" ).grid( row = 6, column= 1 )
        de_rate1 = Entry( self.update_frame )
        de_rate1.grid(row = 7, column = 1)
        de_rate1.insert(0,"0.11")

        Label( self.update_frame, text="Equity" ).grid( row = 8, column= 0 ) 

        def cal_rmbs():
            self.period = int(term.get())
            self.mortgage_cf = []
            #cb_mortgage_type.select()
            smm = 1 - math.pow(( 1 - float(self.rmbs_cpr.get())),1/12.0)
            #print smm
            #print "SMM %f" % smm
            balance = float(e.get())
            #print balance
            period = int(term.get())
            #mortgage_rate = 

            r = float(mortgage_rate.get())/12.0
            #print self.rmbs_mortgageType.get()
            if self.rmbs_mortgageType.get() == 1 :
            #fix principal method
                monthly_principal = balance / float(period)
                for i in range(period):
                    mcf = 0.0
                    mcf += monthly_principal
                    mcf += balance * r
                    mcf += balance * smm
                    if balance >= monthly_principal:
                        balance -= monthly_principal
                    else:
                        break
                    if balance >= balance * smm:
                        balance -= balance * smm
                    else:
                        break
                    self.mortgage_cf.append(mcf)
            #fix amount method
            else:
                mthpyt = balance * r * math.pow((1+r),period)/ (math.pow((1+r),period) -1)
                for i in range(period):
                    mcf = 0.0
                    interest = balance * r
                    mcf += mthpyt
                    mcf += balance * smm

                    if balance >= (mthpyt - interest):
                        balance -= mthpyt - interest
                    else:
                        break
                    if balance >= balance * smm:
                        balance -= balance * smm
                    else:
                        break

                    self.mortgage_cf.append(mcf)

            #self.mortgage_cf = x for x in self.mortgage_cf if x > 0

            self.senior_cf = []
            self.junior_cf = []
            self.equity_cf = []
            senior_req = float(de.get()) * float(de_rate.get())
            junior_req = float(de1.get()) * float(de_rate1.get())
            for i in self.mortgage_cf:
                if i <= senior_req:
                    self.senior_cf.append(i)
                    self.junior_cf.append(0)
                    self.equity_cf.append(0)
                elif i <=  senior_req+junior_req:
                    self.senior_cf.append(senior_req)      
                    self.junior_cf.append(i - senior_req)
                    self.equity_cf.append(0)
                else:
                    self.senior_cf.append(senior_req)
                    self.junior_cf.append(junior_req)
                    self.equity_cf.append(i-senior_req-junior_req )

            self.rmbs_render()

        Button( self.update_frame, text="Render", command = cal_rmbs).grid( row = 8, column= 1 )

    def rmbs_render(self):
        #print self.mortgage_cf
        #print self.senior_cf
        #print self.junior_cf
        #print self.equity_cf

        plt.cla()
        x = np.arange(0.0, len(self.mortgage_cf), 1)

        from pylab import *

        plot(x, self.mortgage_cf,color='#ff9999',linewidth=2.0, label='Total MBS Cash Flow')
        plot(x, self.equity_cf,color='#BB1BF5',linewidth=2.0, label='Equity Tranche Cash Flow')
        plot(x, self.junior_cf,color='#2340E8',linewidth=2.0, label='Junior Tranche Cash Flow')
        plot(x, self.senior_cf,color='#1DF0A2',linewidth=2.0, label='Senior Tranche Cash Flow')

        legend( ('Total MBS Cash Flow', 'Equity Tranche Cash Flow', 'Junior Tranche Cash Flow','Senior Tranche Cash Flow'), loc='upper right')
        #plt.fill_between(x, self.mortgage_cf,0, 'r')
        plt.grid(True)
        plt.show()

A Walk-through on Modeling MBS/Tranche in C#

Modeling on MBS/Tranche involves 3 entities . Mortgages , Mortgage Pool and Tranches . Mortgage Pool is key in this relationship . Mortgage Pool collects cash flows from Mortgages and redistribute into different Tranches.

Class MortgagesPool

One Mortgage Pool has many Mortgages and One Mortgage Pool has many Tranches. So , the class of Mortgage Pool may like this :

class MortgagesPool{
  public ArrayList tranches = new ArrayList();
  public ArrayList mortgages = new ArrayList();
// add a mortgage to mortgage pool
  public void addMortgage(Mortgage m){
  mortgages.Add(m);
 }
// add a tranche to mortgage pool
  public void addTranches(Tranche t){
  tranches.Add(t);
 }
// distribute cash flow to tranches
  public void distributeCashFlow(){
.........}

I’ve setup  2 ArrayList to hold tranches and mortgages in a single instance of MortgagesPool . and implement 2 methods to add tranches and mortgages into an instances of MortgagesPool. “distributeCashFlow()” is used to calculate cash flows received by Tranches in the mortgages pool and assign the cash flow array to the instances of tranches .

Class Mortgage

class Mortgage{
 //cacluate payment via fix-principle method 
  public float amount;
  public float mortgageRate; // monthly interest rates
  public int paymentTimes; // number of months to pay the mortgage
  public float[] monthlyCashFlow; // monthly payment for mortgage
public Mortgage(float amount, float mortgageRate, int paymentTimes){
  monthlyCashFlow = new float[paymentTimes];
  this.amount = amount;
  float outstandingAmount = this.amount;
  this.mortgageRate = mortgageRate;
  this.paymentTimes = paymentTimes;
  for (int i = 0; i < paymentTimes; i++){
     monthlyCashFlow[i] = this.amount / paymentTimes + outstandingAmount * this.mortgageRate;
     outstandingAmount -= this.amount / paymentTimes;
  }
 }
public void getPayments(){
  foreach (float p in this.monthlyCashFlow){
     Console.Write(p);
     Console.Write(" ");
  }
}
}

To simple the calculation of mortgage payments , I just apply the fix principle repayment method for this mortgage.

Class Tranche

class Tranche{
  public int priority; // 1 for highest and 99 for equity tranche
  public float returnRate;
  public float notionalAmount;
  public float[] actualCashFlow; // actualCashFlow received
  public Tranche(int priority, float returnRate, float nontionalAmount) {
    this.priority = priority;
    this.returnRate = returnRate;
    this.notionalAmount = nontionalAmount;
  }
 }

In Tranche class, the key element is “priority” member . this is will be used when MortgagesPool distribute out cash flow collected by mortgages. Tranche with less priority number will have a higher priority in fulfilling its cash flow requirement .

Distribute Cash Flow

in MortgagesPool Class, distribute cash flow function is omit because it is a little complex:

public void distributeCashFlow(){
    int maxPeriod = 0;
    //get maxPeriod among all mortgages
    foreach (Mortgage m in mortgages){
        if (m.monthlyCashFlow.Length > maxPeriod){
            maxPeriod = m.monthlyCashFlow.Length;
        }
    }

    //Initial tranches cash flow array
    foreach (Tranche t in tranches){
        t.actualCashFlow = new float[maxPeriod];
    }

    for (int i = 0; i < maxPeriod; i++){
        float paymentForCurrentPeriod = 0f;
        // collect all cash flow from mortgages in the pool
        foreach (Mortgage m in mortgages){
            if (i < m.paymentTimes){
                paymentForCurrentPeriod += m.monthlyCashFlow[i];
            }
        }

        int tmpPriority = 1;
        //distribute cash flow to tranches according priority
        foreach (Tranche t in tranches){
            if (t.priority == tmpPriority) {
                if (t.notionalAmount * t.returnRate <= paymentForCurrentPeriod) {
                    t.actualCashFlow[i] = t.notionalAmount * t.returnRate;
                    paymentForCurrentPeriod -= t.notionalAmount * t.returnRate;
                }else{
                    t.actualCashFlow[i] = paymentForCurrentPeriod;
                    paymentForCurrentPeriod = 0f;
                }
                tmpPriority++;
                continue;
            }
        }

    }
}

Example with 1 /1

We first setup a simple scenario, 1 mortgage ,1 tranche:

static void Main(string[] args)
{
    Mortgage m = new Mortgage(400000f, 0.012f, 36);
    Tranche t = new Tranche(1, 0.02f, 600000);
    MortgagesPool mp = new MortgagesPool();
    mp.addMortgage(m);
    mp.addTranches(t);
    mp.distributeCashFlow();
    Console.WriteLine("Cash Flow from Mortgage");
    m.getPayments();
    Console.WriteLine(" ");
    Console.WriteLine("Cash Flow received by Tranche");
    // print out how much cash flow received by Tranche
    foreach (float cf in t.actualCashFlow){
        Console.Write(cf);
        Console.Write(" ");
    }
    Console.ReadKey();
}

Result:

newscreen189769101

We can see that :

1) in earlier payments that greater than tranche require, tranche only received its own quota.

2) in later payments that less than tranche need, tranche only received residual payments.

Example with 2 /1

Now , we try to setup 2 mortgages and 1 tranche MBS.

static void Main(string[] args){
    Mortgage m = new Mortgage(400000f, 0.012f, 36);
    Mortgage m2 = new Mortgage(400000f, 0.012f, 36); // add second mortgage
    Tranche t = new Tranche(1, 0.02f, 600000);
    MortgagesPool mp = new MortgagesPool();
    mp.addMortgage(m);
    mp.addMortgage(m2);
    mp.addTranches(t);
    mp.distributeCashFlow();
    Console.WriteLine("Cash Flow from Mortgage 1");
    m.getPayments();
    Console.WriteLine("Cash Flow from Mortgage 2"); // add second mortgage
    m2.getPayments();
    Console.WriteLine(" ");
    Console.WriteLine("Cash Flow received by Tranche");
    // print out how much cash flow received by Tranche
    foreach (float cf in t.actualCashFlow){
        Console.Write(cf);
        Console.Write(" ");
    }

    Console.ReadKey();
}

Result:

newscreen190073427
Now , After add another mortgage into the pool , all tranche requirement are fulfilled completely . Cheers !

Bond Calcuation with C#

I’ve been read a tutorial about C# in last 2 days . Before that, I haven’t read a single line of C# code in since I started to programming.  C# syntax is very similar to C++ and Java . But develop in C# is not easy as its syntax seems.  I’ve wrote some trivial codes about some financial calculation of a bond, it’s quite similar to the code I’ve written before in Python, but this one focus  more  in a risk neutral perspective.

Basic Bond Class with No Coupon

class ZeroBond{
 protected float price; // current price for bond
 protected float riskFreeRate; 
 protected float faceValue;
// constuctor
 public ZeroBond(float price, float riskFreeRate, float faceValue){
 this.price = price;
 this.riskFreeRate = riskFreeRate;
 this.faceValue = faceValue;
 }}

 Value a Non-Coupon Bond with  Risk Neutral Binomial Method

I won’t go to detail about what is a Risk Neutral  or what’s the Binomial Method . In short , a risky bond equals to short position on put plus a risk-free bond .

$$B(t,T) = B_{risk free}(t,T)-P(t,T,K)$$

To estimate the value of bond $$B(t,T)$$, it is easy to get the data of risk free bond ,like Treasury Bond . but , how to evaluate the value of $$P(t,T,K)$$ ?

Price of a Put given Risk Neutral Binomial Method. $$pi$$ is the probability of upward change of price and K is the face value of bond , $$V_{d}left ( 1 right )$$ is the downward price of the bond and this value can be access via multiply a change rate with initial price of bond $$V_{0}$$ . Question remains that : how to estimate the probability of upward change of price ? the value of $$pi$$ ?

$$P(0) = frac{1}{1+r}left [ pi * 0 + left ( 1-pi right )*left ( K-V_{d}left ( 1 right ) right ) right ]$$

We can infer the value of $$pi$$ by assumption that current price of bond is present value of expectation of future 2 prices( upward&downward ). If we input current price, estimation of change rate and risk free rate, then we will get $$pi$$ probability of upward price changing.

$$V(0)=frac{1}{1+r}(pi * V_{u}(1)+(1-pi)*V_{d}(1))$$

C# Code :

public float CalRiskNetrualProbWithBinomialOption(float ChangeRate)
 {
 // calculate risk neutrual probability for upward of prices
 return (this.price * (1 + this.riskFreeRate) - this.price * (1 - ChangeRate)) /
 (this.price * (1 + ChangeRate) - this.price * (1 - ChangeRate));
 }
public float CalPutWithBinomialOption(float ChangeRate){
 // calculate 1 period bond put option price with binomial change price
 float upwardProb = this.CalRiskNetrualProbWithBinomialOption(ChangeRate);
 return (1 - upwardProb) * (this.faceValue - this.price * (1 - ChangeRate)) / (1 + this.riskFreeRate);
 }

After we have the put price for the risky bond , we now may calculate the risky bond price by discount risk free bond minus the put price .

public float CalRiskNetrualPriceWithBinomialOption(float ChangeRate) {
 // calculate risk neutral price with Binomial Option 
 return this.faceValue / (1 + this.riskFreeRate) - this.CalPutWithBinomialOption(ChangeRate);
 }

Default Bond with a Recovery

Consider a risky zero-coupon, one-period debt with the face value of K (i.e., promising a cash
flow of K at maturity in one period). Given the expected recovery for this bond in case of default, R,
then the bond has an expected payoff of K × R in default with the probability λ (the probability of
default) and, of course, a payoff of K in the absence of default with the probability of (1 – λ). (From CAIA text book , page:631). A risky bond value can be expressed as the sum of the discounted cash flows as below:

$$B(0,1)=lambda *frac{K * R}{(1+r)} + (1-lambda)*frac{K}{(1+r)}$$

Thus , we may input a recovery rate and then get the price of risky bond:

public float CalRiskNetrualPriceWithRecovery(float RecoveryRate, float DefaultProb){
 return DefaultProb * this.faceValue * RecoveryRate / (this.riskFreeRate + 1) +
 (1 - DefaultProb) * this.faceValue / (this.riskFreeRate + 1);
 }

Basic Bond Class with Coupon

The class Bond inherits the “ZeroBond”, and has its own member like “interestRate”,”PaymentsPerYear”,”TimeToMaturity” etc

class Bond : ZeroBond{
 private float interestRate; // Interest Rate for Bond
 private int paymentsPerYear; // #Only implement "1" for now # 1 for interest paid yearly and 2 for interest paid semi-yearly
 private float term; // Maturity Term
 private float timeToMaturity;
public Bond(float price, float riskFreeRate, float faceValue , float interestRate, int paymentsPerYear, float term, float timeToMaturity)
 : base(price, riskFreeRate, faceValue){
 this.interestRate = interestRate;
 this.paymentsPerYear = paymentsPerYear;
 this.term = term;
 this.timeToMaturity = timeToMaturity;
 }
}

 

Calculate Present Value (continuously compound)

I wrote two method to get a float array of payment dates and cash flow , then using Math.E , Math.pow() to discount the cash flow to present value.

// Cash flows including interest and principle at last payment
 public float[] GetCashFlow(){
 int timesOfPayments = (int)Math.Ceiling(this.timeToMaturity * this.paymentsPerYear);
 float[] cf = new float[timesOfPayments];
 float interest = this.interestRate * this.faceValue;
 for(int i = 0; i < timesOfPayments; i++){
 cf[i] = interest;
 }
 cf[ timesOfPayments-1 ] += this.faceValue;
 return cf;
 }
//cash flow payment dates
 public float[] GetTerm(){
 float ytm = this.timeToMaturity;
 return Enumerable.Range(0, (int)Math.Ceiling(ytm)).Select(i => (float)Math.Round(i + (ytm - (int)ytm), 1)).ToArray();
 }
// caluclate Present Value
 public float CalPresentValue(float discountRate){
 float[] term = this.GetTerm();
 float[] cfs = this.GetCashFlow();
 float presentValue = 0f;
 for (int j = 0; j < cfs.Length; j++){
 presentValue += (float)(Math.Pow(Math.E, -1 * term[j] * discountRate) * cfs[j]);
 }
 return presentValue;
 }

Calculate Yield To Maturity

Yield to Maturity is a discount rate that will equals the present value of cash flow to the current price of the bond . I wrote a bi-sect method to calculate this rate . “maxIteration” is set with 1000 up to prevent infinite looping and this method assume that discount rate between 0.001 to 0.999 .

public float CalYield(float eps){
 float des = 0.0f;
 const int maxIteration = 1000;
 int iterNum = 0;
 float y = 0.04f;
 float h = 0.999f;
 float l = 0.001f;
 do{
 des = this.CalPresentValue(y) - this.price;
 if (des > 0){
 l = y;
 y = (y + h) / 2;
 }else{
 h = y;
 y = (y + l) / 2;
 }
 //Console.WriteLine(Math.Abs(des));
 iterNum++;
 }while ((Math.Abs(des) > eps) && (iterNum <= maxIteration));
 return y;
 }

Calculate Macaulay  Duration

Macaulay duration is the weighted average maturity of cash flows with formula below :

$$MacDuartion = sum_{i=1}^{n}t_{i}frac{PV_{i}}{V}$$

$$PV_{i}$$ stands for the present value of cash flow at time i , V stands for the current price.

// Calculate Macaulay  Duration
 public float CalDuration(float discountRate){
 float PV = this.CalPresentValue(discountRate);
 float[] term = this.GetTerm();
 float[] cfs = this.GetCashFlow();
 float duration = 0f;
 for (int i = 0; i < term.Length; i++){
 duration += term[i] * (float)Math.Pow(Math.E, (-1 * term[i] * this.riskFreeRate)) * cfs[i] / PV;
 }
 return duration;
 }

 Example:

static void Main(string[] args){
 // a zero bond with price 100.00 , risk free rate 0.05 , and face value 80.0
 ZeroBond zero_bnd = new ZeroBond(100.0f,0.05f,80f);
 // single period calculation with 2 possible result;
 Console.WriteLine("Downside Probibility");
 Console.WriteLine(zero_bnd.CalRiskNetrualProbWithBinomialOption(0.3f));
 Console.WriteLine("Put Value");
 Console.WriteLine(zero_bnd.CalPutWithBinomialOption(0.3f));
 Console.WriteLine("Risk Neutral Price");
 Console.WriteLine(zero_bnd.CalRiskNetrualPriceWithBinomialOption(0.3f));
//Bond with current price 103.3, risk free rate 0.05, face value 100
 // coupon rate 0.03, pays yearly with 4 yr term and 3.32 years remains 
 Bond int_bnd = new Bond(103.3f, 0.05f, 100f, 0.03f, 1, 4, 3.32f);
Console.WriteLine(int_bnd.GetCashFlow());
 // calculate present value of all cash flow
 Console.WriteLine("Calculate the Present Value ");
 Console.WriteLine(int_bnd.CalPresentValue(0.05f));
Console.WriteLine("Yield to Maturity, with eps 0.0001 ");
 Console.WriteLine(int_bnd.CalYield(0.0001f));
 // calculation Macauley Duration
 Console.WriteLine("Duration");
 Console.WriteLine(int_bnd.CalDuration(0.02f));
 Console.ReadKey();
 }

Result

newscreen179516933

Conditional Checking in Python

When story begins

In Python , there is a shortcut in conditional checking, I didn’t take much care about the difference of below form of conditional syntax :

if var :
    stmt;
if var is not None :
    stmt;
if var != None:
    stmt;

Explanation behind the scene is an analogy to difference of  an empty string “” and NULL in a database . “”  is an empty string but means something, NULL is nothing means nothing . In Python world ,  None equals NULL in Database , that’s why “” == None will return false .

>>> v = None
>>> if v is not None:  # return false, not print anything
 print "V is not initialized"

>>> v = ""
>>> if v is not None:   # "" is not None, return True
 print "V is not initialized"
# print "V is not initialized"
>>> if v != None:     #  "is not" works same effect as "!="
 print "test V with equality"
# print "test V with equality"

 

but ” if var : ” is little ambiguous,  behind the scene, Python will evaluate var using function bool()

newscreen768477149

Dive a little

since if statement is evaluate the expression using bool() function , bool() function actually treat other SOMETHING EMPTY as False , for example , an emtpy tuple (), an empty list [] etc .

newscreen768905262

Summary

1) “not var” will be true  in “if” statement , when var is (), ”, u”, {}, [], 0, 0.0, 0j

2) var is not None will always be True unless a variable is actually None.

Constraints on Arguments in Python

When Dynamic Type comes to Trouble

As a dynamic programming languages , Python don’t have a type on variables , as well as a arguments . This feature save us a lot workload and it enable us to focus on logical of program .

def add(x, y):
    return x+y

print add(5,6)
>> 11
print add("D","E")
>> "DE"
print add(5.0, 6)
>> 11.0

Above function is straightforward and works fine if x and y are passed with int/ float/ string .

But things may not expect as we thought, what if we pass a list parameter into this function ?

print add([5],6)
Traceback (most recent call last):
 File "C:UsersIBM_ADMINworkspaceeulertest_blog.py", line 4, in <module>
 print add([5],6)
 File "C:UsersIBM_ADMINworkspaceeulertest_blog.py", line 2, in add
 return x+y
TypeError: can only concatenate list (not "int") to list

When Decorator comes to Help

I won’t take too much time explain on how decorator works . Pls refer to Bruce Eckel’s post and Brian Holdefehr’s post . Basically , we need a decorator to constrain the arguments passed into function, if the arguments are not consist with predefined value , we raise an exception and tell the user that parameters that won’t match .

class ArgumentsCheck(object):
    def __init__(self, *argstype):
        self.argstype = argstype # store all parameters of decorater

    def __call__(self, f):
        def wrapped_f(*args):
            # test if all type of params of decoratoer equals to those of functions
            if not all([ type(a) == type(b) for a,b in zip(args, self.argstype) ]) : 
                # Not all equals, then print all type of decoratoer and those of functions
                raise Exception("Invalid Params Type for Function %s,Argument Type required (%s) but a Type Supplied (%s) " % (f.__name__,','.join([ type(_).__name__ for _ in self.argstype]),','.join([ type(_).__name__ for _ in args])) )
            else:
                # if all params type matches ,then call the funtion with params
                f(*args)
        return wrapped_f

Define Parameter types of Function by Decoration

@ArgumentsCheck(0,0)
def func1(num1, num2): #the two params must be integers
    print num1,num2

func1(1, 2) # this will be good
print "==================n"
func1(1, 2.0) # this will fail

Output:

1 2
==================

Traceback (most recent call last):
  File "C:UsersIBM_ADMINworkspaceeulertest_blog.py", line 27, in <module>
    func1(1, 2.0) # this will fail
  File "C:UsersIBM_ADMINworkspaceeulertest_blog.py", line 11, in wrapped_f
    raise Exception("Invalid Params Type for Function %s,Argument Type required (%s) but a Type Supplied (%s) " % (f.__name__,','.join([ type(_).__name__ for _ in self.argstype]),','.join([ type(_).__name__ for _ in args])) )
Exception: Invalid Params Type for Function func1,Argument Type required (int,int) but a Type Supplied (int,float)

Conclusion:

By this decoration, we make Strong Typed  arguments call in Python, even 0 and 0.0 behaves differently in this case .

Download Chinese listed Company’s Balance Sheet from Hexun.com

I’ve written a perl script to fetch Balance Sheet data and export them into a Excel spreadsheet .

Pre-preparation:

use LWP::Simple -> this pacake is for scap data from web
use Excel::Writer::XLSX -> for writing excel file

Download

https://github.com/hhh6897/Hexun.Balance.Sheet

Usage :

perl hx.pl [Stock ID sperated by comma] [start year] [end year] [name of output excel spreadsheet]

Result:

newscreen1267346334

[译文]Decorators and Functional Python[draft]

Original blog post wrote by Brian Holdefehr. This is a Chinese version of original post .
本文原文作者 Brian Holdefehr。若有不足,敬请斧正。

装饰器(Decorators) 是Python众多强大特性之一。装饰器除了本身编程语法上的作用之外,另外提供了一种有趣思维方式——”函数化”思维(a functional way)

我会从头开始介绍介绍装饰器的工作原理,接着会介绍几个必须了解的基本概念。之后,我们会详细深入探索一些装饰器实例以及其工作方式。最后我们会讨论一些装饰器的高阶应用,例如参数化装饰器(optional arguments),以及嵌套装饰器(chaining)。

首先,我们给出最简单的Python函数装饰器的定义。

函数 是一个执行特定任务并且可以复用的代码块。

那么,什么是装饰器呢?

装饰器 是一个修改其他函数的函数

现在,我们从几个基本概念的解释开始,逐渐详细深入装饰器的含义。

函数是对象

在Python世界里,所有一切都是对象。这就意味着说,函数本身可以通过名字对其引用(referred to by name),并且函数和其他对象一样可以进行传递。例如:

def traveling_function():
     print "Here I am!"

function_dict = {
    "func": traveling_function
}

trav_func = function_dict['func']  #存储字字典中的函数可以和普通函数一样调用。
trav_func()
# >> Here I am!

在function_dict 字典(dictionary)中,键(key)”func” 存储着 traveling_function 函数对象。在字典中的这个函数对象可以和普通函数一样进行调用。

高阶函数

函数对象可以像其他对象一样在执行过程中进行传递。我们可以把函数对象存储在字典里,或者把它们放到列表里面,甚至可以把它们赋值给对象属性中(object properies) 。那么,我们能不能让函数像参数一样传递到另外一个函数中?当然可以!如果某函数A接受另外一个函数B作为参数 或者该函数A返回另外一个函数对象C,那么这个函数称之为”高阶函数“。

def self_absorbed_function():
    return "I'm an amazing function!"
def printer(func):
    print "The function passed to me says: " + func()

# Call `printer` and give it `self_absorbed_function` as an argument

printer(self_absorbed_function)
# >> The function passed to me says: I'm an amazing function!

这里你可以看到,函数对象self_absorbed_function可以作为参数传入另外函数printer中。传入之后,函数对象printer可以调用传入的函数对象self_absorbed_function。在了解这种特性之后,我们就可以建立有趣的函数,比如 —- 装饰器!

装饰器初探

本质上,装饰器只是一个接受函数对象A作为参数的函数。在大多数情况下,装饰器会返回修改过的函数对象A’ 。下面简单的装饰器例子可以帮助我们理解其工作方式。

def identity_decorator(func):
    def wrapper():
        func()
    return wrapper
def a_function():
    print "I'm a normal function."
#`decorated_function`存储了`identity_decorator`返回的函数对象, 也是其返回的wrapper返回的对象
decorated_function = identity_decorator(a_function)
# 下面语句调用 `identity_decorator` 所返回的函数对象(因为这里使用了括号对)
decorated_function()
# >> I'm a normal function

这里,identity_decorator 并没有修改其包装(wraps)的函数对象func 。在调用identity_decorator的时候,它仅仅返回函数wrapper执行结果,该wrapper会调用函数identity_decorator的参数func。目前,这装饰器没有实际用途。

identity_decorator这个函数很特别,即便func对象没有作为参数传入到wrapper函数,wrapper函数仍然可以读取 func对象!可是为什么会这样呢?这就是因为”闭包”(closures)。

闭包

闭包是个看似高级的词语,实际意思是当函数声明之后,该函数仍然保留一个指向其诞生环境的引用(reference)。

在前例定义wrapper函数时候,wrapper在其本地作用域(local scope)中读取了func对象。这就意味着在wrapper函数生命周期中(从返回wrapper()函数对象到 函数对象identity_decorator返回到decorated_function为止 ),该wrapper一直可以读取func对象。一旦identity_decorator 函数进行返回之后,唯一读取func对象的方法就是通过 函数对象decorated_function。func 仅仅存在于 decorated_function的闭包环境内。

实例:简单装饰器

现在,我们创建一个装饰器。这个装饰器的功能是记录其所修饰函数对象的调用次数。

def logging_decorator(func):
    def wrapper():
        wrapper.count += 1
        print "The function I modify has been called {0} times(s).".format(
        wrapper.count)
        func()
        wrapper.count = 0
    return wrapper

def a_function():
    print "I'm a normal function."

modified_function = logging_decorator(a_function)
modified_function()

# >> The function I modify has been called 1 time(s).
# >> I'm a normal function.
modified_function()
# >> The function I modify has been called 2 time(s).
# >> I'm a normal function.

之前我们说过,“装饰器会修改函数对象”,这样想往往可以帮助我们理解。但是你可以从我们的例子中看出来:logging_decorator装饰器真正做到事情是“返回”的是一个全新,拥有记录日志功能的类似a_function函数对象。

在这里例子中,logging_decorator 不仅仅接受了一个函数对象作为参数,它同时返回了函数对象”wrapper”。每次logging_decorator返回函数被调用的时候,它会自动对wrapper.count 进行加一并输出该值,然后再调用logging_decorator所包装的函数func(在这里是a_function())。

你可能奇怪为什么我们把wrapper的一个属性(property)作为计数器,而不是用单独的变量作为计数器。wrapper的闭包空间不是给我们读取任何在本地空间(local scope)变量的权限吗?是的,但是这里需要特别注意:在Python,闭包提供了在函数作用范围内读取任何变量的权限,但是只提供向(mutable)对象写的权限。一个整型变量是(immutable)的Python不可变对象,所以我们无法在整型变量上进行累加。因为wrapper对象是可变(mutable)对象,所以我们选择在wrapper对象的属性上进行累加。

装饰器语法

我们在之前的例子中看到,通过函数对象作为参数的传入进行装饰器的使用。实际情况是用装饰器函数包装(wrapping)该传入的函数。然而,当你熟悉装饰器之后,Python 另外有一个更加直观,更为精简的语法(syntax pattern)。

# 在之前例子中,我们向装饰器logging_decorator传入需要修改的函数对象some_function,然后把修改后的函数对象赋予名为some_function的变量。
def some_function():
    print "I'm happiest when decorated."

# 这里,我们把装饰器所返回的函数对象名字赋予原传入函数对象的名字。
some_function = logging_decorator(some_function)
# 可见,传入函数对象名字“some_function”字样重复出现2次略显冗余,优雅的Python提供更为简便表述方式--装饰器语法(decorator syntax)。
@logging_decorator
def some_function():
    print "I'm happiest when decorated."

使用装饰器语法 ,语句执行逻辑如下:

  1. Python解释器读到装饰器函数的时候,首先编译some_function函数 ,并且标记其名字为“some_function”。
  2. 随后把some_function函数对象传入到 @字符后面名为“logging_decorator” 的装饰器函数。
  3. “logging_decorator”装饰器返回修改后的函数对象替代了原来some_function函数对象,并且和名称“some_function”名称进行绑定。

当你记住这3个步骤之后,我们对identity_decorator 进行详细的解释。

def identity_decorator(func):
    # 当装饰器初始化,并且被func对象传入的时候,所有写在这里的语句会被执行(上述第二步)
    def wrapper():
    # 每次最后包装的函数返回对象被调用时,这里语句在会被执行
        func()
    return wrapper

希望这些注释足够明了。在包装函数wrapper所返回对象调用时候,其wrapper函数体内的命令都会被执行。在wrapper函数外,装饰器identity_decorator 内的语句只会在装饰器被第一次传入参数的时候会被调用(即第二步)

在继续了解装饰器之前,我还需要解释*args 和**kwargs 用法。

*args 和 **kwargs

你以前可能经常看见这两个修饰符同时出现,现在我们将对其一一进行讲解。

  • Python函数可以通过*args传入任意个数的位置参数(positional arguments), *args 会把所有匿名参数压缩到一个元组(tuple)中。函数可以访问该元组的所有成员。相反地,当调用函数使用*args时候, *args 变量将会自己解压成匿名参数列表。
def function_with_many_arguments(*args):
    print args
# args在函数体内是一个由传入多个参数所组成的元组,该元组可以和其他元组一样被函数体读取

function_with_many_arguments('hello', 123, True)
# >> ('hello', 123, True)
def function_with_3_parameters(num, boolean, string):
    print "num is " + str(num)
    print "boolean is " + str(boolean)
    print "string is " + string
arg_list = [1, False, 'decorators']

# 通过在arg_list加上* 之后,arg_list将会自动解压缩3个位置参数
function_with_3_parameters(*arg_list)

# >> num is 1 
# >> boolean is False 
# >> string is decorators
  • 获取列表内容, 在定义函数后面括号里的参数*args 会压缩一系列位置参数成为名字为‘args’的元组变量。在函数体内元组*args 会包含一系列位置参数供函数调用。
  • 你可以看到参数展开的例子, * 符号不仅仅可以修饰‘args’,其他名字也同样适用。我们这里只是作为约定俗成地命名‘args’所有位置参数用。
  • **kwargs 和*args 的作用是类似的,但是**kwargs 不是压缩/解压 位置参数,而是 压缩/解压 命名参数。 如果**kwargs 出现在函数的参数列表里面,它会把所有可能的命名参数都压缩到一个字典里面。
def function_with_many_keyword_args(**kwargs):
    print kwargs
function_with_many_keyword_args(a='apples', b='bananas', c='cantalopes')
# >> {'a': 'apples', 'b': 'bananas', 'c': 'cantalopes'}
def multiply_name(count=0, name=''):
    print name * count
arg_dict = {'count': 3, 'name': 'Brian'}
multiply_name(**arg_dict)
# >> BrianBrianBrian

你现在明白*args 和 **kwargs 的魔法作用。现在我们看看装饰器的有用之处。

缓存(Memoization)

缓存是避免冗余的计算开销。你可以将每次函数返回的结果暂时存储起来,通过缓存这个方法,如果下次对改函数传入相同的参数,该函数就不必重新花时间再去计算一遍,而是直接返回在缓存中已经存在的结果。

from functools import wraps
def memoize(func):
    cache = {}
    @wraps(func)
    def wrapper(*args):
        if args not in cache:
           cache[args] = func(*args)
        return cache[args]
    return wrapper

@memoize
def an_expensive_function(arg1, arg2, arg3):
 ...

你可能已经注意到这里@wraps 奇怪的用法。在讨论缓存之前我会扼要地简述这个看似奇怪的wraps用法。

  • 装饰器的一个副作用就是被包装的函数失去了其原本的3个属性,__name__, __doc__以及__module__。wraps函数 包装了装饰器所需要返回的函数,保留了这三个属性,如同包装函数没有修改过他们一样。例如:如果不是用wraps函数进行包装的话,an_expensive_function名字就可能变成‘wrapper’,

我认为 缓存 诠释了装饰器好用处之一。这能满足众多函数希望提升性能的需求,并且 如果创建一个灵活(generic)的装饰器,我们可以用它来 装饰 其他不同函数,这样其他函数都可以从中获得速度的提升。这就避免了在不同地方为不同函数分别造轮子。通过这样的抽象,我们的代码会更加易于维护,同事更加易于阅读和理解。之后,如果你看到一个@开始的单词之后,马上明白这个函数已经被缓存了。

我应该注意到 缓存仅仅适用于纯函数(pure functions)。所谓纯函数就是 “当参数是一定时候,该函数永远返回相对应的值”。是不是纯函数,这个取决于全局变量是否参与了函数体内的计算,或者取决于系统的输入输出,或者其他东西影响到该函数的返回值。缓存在这些外接变量影响下会返回和函数真实情况不一致的值。同样,一个纯函数不会有任何副作用。所以如果你的函数对计数器进行累加,或者调用方法,或者调用其他对象,只要这些不会影响到这个函数的返回值,那么缓存返回的值副作用不会发生。

类装饰器

原本我们说装饰器是一个修改函数的函数。但是,他们同时可以用来修改类或者方法。装饰器很少用来装饰类,但是它可以在一些特定的场景下替代 metaclasses(译者:metaclasses在99%的场景下是没有什么用处的)。

foo = ['important', 'foo', 'stuff']
def add_foo(klass):
    klass.foo = foo
    return klass

@add_foo
class Person(object):
    pass

brian = Person()
print brian.foo

# >> ['important', 'foo', 'stuff']

现在,所有 Person类下面所有对象都会有一个“Foo”属性。注意,由于我们装饰了一个类,该装饰器并不是返回了一个函数对象,而是返回了一个类对象。于是,在这里我们拓展了装饰器的定义:

装饰器是 修改函数,方法(methods)或者类(classes)的函数。

装饰器是可调用的类

事实证明,我之前对你有所隐瞒。装饰器不仅仅可以装饰类,装饰器本身也可以成为一个类!唯一需要满足要求:装饰器的返回值必须是可以调用的(callable)。这意味着装饰器的返回值必须定义 默认的”__call__()”方法。”__call__()”方法 即为你调用对象时,该被调用对象所执行的内容。当然,函数对象都会自动定义了这个__call__() 方法。现在,我们重新建立identity_decorator类, 看看它是如何工作的。

class IdentityDecorator(object):
    def __init__(self, func):
        self.func = func
    def __call__(self):
        self.func()

@IdentityDecorator
def a_function():
    print "I'm a normal function."
a_function()
# >> I'm a normal function

以下是执行过程:

  • 当IdentityDecorator 装饰 a_function时候,它像装饰器函数一样。这段代码等同于: a_function= IdentityDecorator(a_function)。这个装饰器类通过传入函数对象进行初始化。
  • 当IdentityDecorator 实例化后, IdentityDeorator的初始化函数函数__init__()接受需要装饰的函数对象。在这个例子中,这个初始化函数是接受函数对象,并且把它添加到IdentityDecorator对象属性中,这样之后该函数对象就能够被其他函数访问。
  • 最后,当a_function (由IdentityDecorator对象返回包装过的a_function)被调用的时候,该对象的__call__()方法将会运行。

现在,我们再次拓展装饰器的定义:

装饰器是一个可调用的对象,它能修改函数对象,方法(methods),和类。

参数化的装饰器

有时候你需要针对不同情况,针对性地需要改变修饰器行为。这种修改可以通过传递参数来实现这种变动。

from functools import wraps
def argumentative_decorator(gift):
    def func_wrapper(func):
        @wraps(func)
        def returned_wrapper(*args, **kwargs):
            print "I don't like this " + gift + " you gave me!"
            return func(gift, *args, **kwargs)
        return returned_wrapper
    return func_wrapper

@argumentative_decorator("sweater")
def grateful_function(gift):
    print "I love the " + gift + "! Thank you!"

grateful_function()

# >> I don't like this sweater you gave me!
# >> I love the sweater! Thank you!

我们看看如果不用装饰器语法情况下,不带参数和带参数的修饰器函数各自写法:

# 无参数形式的装饰器
grateful_function = argumentative_function(grateful_function)
# 带有参数形式的装饰器
grateful_function = argumentative_decorator("sweater")(grateful_function)

这里需要关注的是: 当给定一个参数后,装饰器首先会随同参数一起调用(就是 argumentative_decorator 和”sweater”),包装函数(grateful_function)不同于以往那样会直接作为参数传入argumentative_decorator,而是直接和包装函数中的参数func 进行绑定。

newscreen85185297

步步分解:

  1. 解释器执行到被装饰的函数时候,对grateful_function进行编译(如图,将执行“print gift 语句”), 并且将被装饰的函数和名称“grateful_function”进行绑定。
  2. 随后”argumentative_decorator”会被调用,然后传递参数“sweater”到gift,最后它返回‘func_wrapper’函数对象。
  3. ‘func_wrapper’ 会把grateful_function函数作为参数传入,执行语句,func_wrapper将会返回 returned_wrapper函数对象。
  4. 最终,returned_wrapper 将会替代原来的函数对象grateful_function,然后其对象将会和“grateful_function”名称进行绑定。

我认为这些代码比无参数的装饰器要略难于理解, 但是如果你花时间去思考联系一下,应该能够像明白是怎么一回事。

可选参数的装饰器

装饰器接受可选参数的方法有很多。至于用什么,取决于你是否想要使用位置参数(positional arguments),命名参数(keyword arguments),或者是同时适用两者。这里展示一个可选命名参数的装饰器示例。

from functools import wraps

GLOBAL_NAME = "Brian"
def print_name(function=None, name=GLOBAL_NAME):
    def actual_decorator(function):
        @wraps(function)
        def returned_func(*args, **kwargs):
            print "My name is " + name
            return function(*args, **kwargs)
        return returned_func
    if not function: # User passed in a name argument
        def waiting_for_func(function):
            return actual_decorator(function)
        return waiting_for_func
    else:
        return actual_decorator(function)
@print_name
def a_function():
    print "I like that name!"

@print_name(name='Matt')
def another_function():
    print "Hey, that's new!"

a_function()
# >> My name is Brian
# >> I like that name!
another_function()
# >> My name is Matt
# >> Hey, that's new!

如果我们传入命名参数”name”到”print_name“装饰器,那么该函数的行为和之前argumentative_decorator类似。首先,带有参数name的”print_name”装饰器会调用。然后该装饰器”print_name”返回的函数对象会传入到”print_name”函数所包装的函数对象”a_function”

如果我们不提供参数name的话,print_name 就会和无参数装饰器一样。它仅仅会把其包装的函数作为唯一的参数。

装饰器print_name 考虑了这两种情况。它会检查是否接受了一个包装函数参数。如果没有包装函数参数,那么它会返回waiting_for_func函数对象,该对象会将其包装函数作为参数调用。如果有包装函数参数,那么它跳过中间步骤,直接立即调用actual_decorator 函数。

装饰器的嵌套

我们探索最后一个修饰器的特性:嵌套(Chaining)。你可以在特定函数上放置一个以上的装饰器。这语法类似于类的多重继承一样,可以建立一个“多重继承”的函数对象。

@print_name('Sam')
@logging_decorator
def some_function():
    print "I'm the wrapped function!"
some_function()
# 输出如下
# >> My name is Sam
# >> The function I modify has been called 1 time(s).
# >> I'm the wrapped function!

当你嵌套装饰器的时候,他们嵌套的顺数是由下而上。被包装的函数“some_function”最先会被传递到其定义处上面最近的装饰器(这里也就是logging_decorator)。然后经过第一个装饰器作用之后,返回一个函数对象又再一次被最近的装饰器(这里也就是print_name)装饰。如此反复知道所有嵌套的装饰器执行完毕。

由于我们使用的这两个装饰器首先打印了一个值,然后运行被传入的函数对象。这意味着,当被包装的函数被调用的时候,嵌套过程中最后一个装饰器“print_name”将会最早打印出第一行输出。

小结

我认为装饰器最大的好处之一就是他们使你可以用更为抽象地思维方式进行思考。如果你在开始检查一个函数,发现这个函数有一个memoize装饰器的时候,你会立即明白:你在监视一个memoized的函数。如果函数体内包含了memoization code,那么需要你的大脑额外去理解一下,并且总结出可能的缺陷。使用装饰器同样可以让代码复用更加简便同时也节约了时间,让debugging和重构更加简单。

捣鼓装饰器 是学习函数式编程概念(比如高阶函数和闭包)非常好的途径。我希望此文能对大家有所启示。

Get Stock Quote Price API from Yahoo Finance via Python

Difference against Google Finance / Sina.com

Yahoo provide more rich function regarding to the Stock API than Google / Sina.com does. particular ,it provide customize return format which customized by supply  a parameter in calling URL . Full parameter specification pls refer to this link .(credit to   ) .

  • 200 calls per second , you will be warned if you exceed this limits
  • csv file returned
  • No market depth as SIna.com provide
  • Provide stock indication such as “ Earnings per Share”"50 days Moving Average”

Construct Wrapper in Python

Wrapper in Python of Yahoo Finance API is more complicate . The key parameter is Stock Symbol and retrieve parameter . As yahoo api support retrieve multiple stocks in a single call , I add “add/remove” stock function to Tick class .

import urllib

class Tick():
    def __init__(self):
        self.sym_list = []
        self.params = ""

    def add_stock(self, sym):
        if sym in set(self.sym_list):
            self.sym_list.remove(sym)
        self.sym_list.append(sym) 

    def remove_stock(self, sym):
        self.sym_list.remove(sym)

    def fetch(self): 
        str_sym_list = '+'.join(self.sym_list)
        str_param = self.params
        f = urllib.urlopen("http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s" % (str_sym_list, str_param) ).read()
        rlt = f.split(",")
        return  rlt

Now we add a stock symbol to this instance ,as I’ve add a remove function in method “add_stock” to ensure that user can’t assign a existing stock symbol to instance .

>>> t  = Tick()
>>> t.add_stock ("GOOG")
>>> t.sym_list
['GOOG']
>>> t.add_stock("AAPL")
>>> t.sym_list
['GOOG', 'AAPL']
>>> t.add_stock("GOOG") # to ensure that won't duplicate stock symbol in the list
>>> t.sym_list
['AAPL', 'GOOG']

Using Fetch() function to retrieve data from Yahoo API.

>>> t.params = "nab"
>>> t.fetch()
['"Apple Inc."', '515.60', 'N/Arn"Google Inc."', '722.88', 'N/Arn']

Pls noted that , each security spited by “rn” which is default symbol of “New Line “ in Excel Spread sheet .

Further Work

of course, this blog  is a rough script, not translate return string to number , not aggregate data in stocks, not implement a verbose human friendly parameter. But do serve as a tutorial on how to call Yahoo Finance API via Python . :)