pull down resistor

im using lpc1768
i connected button to P0.5, led to P0.4
initially the button pin should go low, if i press the button P0.5 should get high
but the pin is always high
i checked the pin with multimeter without connecting button it is giving 3.3v
here is my code

#include "lpc17xx.h"
#define led (1<<4);
#define button (1<<5)
uint32_t buttonstate;
int main(void)
{
        SystemInit();
        LPC_GPIO0->FIODIR |= led;  //output
        LPC_GPIO0->FIODIR &= ~(button); // input
        LPC_PINCON->PINSEL0 &=~((1<<9)|(1<<8));      //led pin gpio function
        LPC_PINCON->PINSEL0 &=~((1<<11)|(1<<10)); //button pin gpio function
        LPC_PINCON->PINMODE0 |= ((1<<11)|(1<<10)); //button pin pulldown mode function
        while(1)
        {
                buttonstate = ((LPC_GPIO0->FIOPIN & button)>> 5);
                if(buttonstate)
                {
                        LPC_GPIO0->FIOSET |= led;
                }
                else
                {
                        LPC_GPIO0->FIOCLR |=led;
                }
        }
}