﻿<?xml version="1.0" encoding="utf-8"?>
<InstanceReport xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
  <Version>2.2.0.7</Version>
  <hasSegments>false</hasSegments>
  <ReportName>Summary of Significant Accounting Policies</ReportName>
  <ReportLongName>10201 - Disclosure - Summary of Significant Accounting Policies</ReportLongName>
  <DisplayLabelColumn>true</DisplayLabelColumn>
  <ShowElementNames>false</ShowElementNames>
  <RoundingOption />
  <HasEmbeddedReports>false</HasEmbeddedReports>
  <Columns>
    <Column>
      <LabelColumn>false</LabelColumn>
      <Id>1</Id>
      <Labels>
        <Label Id="1" Label="6 Months Ended" />
        <Label Id="2" Label="Jun. 30, 2010" />
      </Labels>
      <CurrencyCode>USD</CurrencyCode>
      <FootnoteIndexer />
      <hasSegments>false</hasSegments>
      <hasScenarios>false</hasScenarios>
      <Segments />
      <Scenarios />
      <Units>
        <Unit>
          <UnitID>Unit12</UnitID>
          <UnitType>Standard</UnitType>
          <StandardMeasure>
            <MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema>
            <MeasureValue>USD</MeasureValue>
            <MeasureNamespace>iso4217</MeasureNamespace>
          </StandardMeasure>
          <Scale>0</Scale>
        </Unit>
        <Unit>
          <UnitID>Unit1</UnitID>
          <UnitType>Standard</UnitType>
          <StandardMeasure>
            <MeasureSchema>http://www.xbrl.org/2003/instance</MeasureSchema>
            <MeasureValue>shares</MeasureValue>
            <MeasureNamespace>xbrli</MeasureNamespace>
          </StandardMeasure>
          <Scale>0</Scale>
        </Unit>
        <Unit>
          <UnitID>Unit13</UnitID>
          <UnitType>Divide</UnitType>
          <NumeratorMeasure>
            <MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema>
            <MeasureValue>USD</MeasureValue>
            <MeasureNamespace>iso4217</MeasureNamespace>
          </NumeratorMeasure>
          <DenominatorMeasure>
            <MeasureSchema>http://www.xbrl.org/2003/instance</MeasureSchema>
            <MeasureValue>shares</MeasureValue>
            <MeasureNamespace>xbrli</MeasureNamespace>
          </DenominatorMeasure>
          <Scale>0</Scale>
        </Unit>
      </Units>
      <CurrencySymbol>$</CurrencySymbol>
    </Column>
  </Columns>
  <Rows>
    <Row>
      <Id>5</Id>
      <Label>Summary of Significant Accounting Policies</Label>
      <Level>3</Level>
      <ElementName>us-gaap_SignificantAccountingPoliciesTextBlock</ElementName>
      <ElementPrefix>us-gaap</ElementPrefix>
      <IsBaseElement>true</IsBaseElement>
      <BalanceType>na</BalanceType>
      <PeriodType>duration</PeriodType>
      <ShortDefinition>No definition available.</ShortDefinition>
      <IsReportTitle>false</IsReportTitle>
      <IsSegmentTitle>false</IsSegmentTitle>
      <IsSubReportEnd>false</IsSubReportEnd>
      <IsCalendarTitle>false</IsCalendarTitle>
      <IsTuple>false</IsTuple>
      <IsAbstractGroupTitle>false</IsAbstractGroupTitle>
      <IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow>
      <IsEquityAdjustmentRow>false</IsEquityAdjustmentRow>
      <IsBeginningBalance>false</IsBeginningBalance>
      <IsEndingBalance>false</IsEndingBalance>
      <IsReverseSign>false</IsReverseSign>
      <PreferredLabelRole />
      <IsEPS>false</IsEPS>
      <FootnoteIndexer />
      <Cells>
        <Cell>
          <Id>1</Id>
          <ShowCurrencySymbol>false</ShowCurrencySymbol>
          <IsNumeric>false</IsNumeric>
          <IsRatio>false</IsRatio>
          <DisplayZeroAsNone>false</DisplayZeroAsNone>
          <NumericAmount>0</NumericAmount>
          <RoundedNumericAmount>0</RoundedNumericAmount>
          <NonNumbericText>&lt;div&gt;
&lt;p style="margin-top: 18px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;2. Summary of Significant Accounting Policies &lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px; margin-left: 2%;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;&lt;i&gt;Basis of Presentation and Use of Estimates &lt;/i&gt;&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;The accompanying unaudited financial statements have been prepared in accordance with U.S. generally accepted accounting principles ("GAAP") for interim financial information and with the instructions to Form 10-Q and Rule&amp;nbsp;10-01 of Regulation&amp;nbsp;S-X. Accordingly, they do not include all of the information and footnotes required by GAAP for complete financial statements. In the opinion of management, all adjustments (consisting of normal recurring accruals) considered necessary for a fair presentation have been included. Operating results for the three and six months ended June&amp;nbsp;30,&amp;nbsp;2010 are not necessarily indicative of the results that may be expected for the calendar year ending December&amp;nbsp;31,&amp;nbsp;2010. These unaudited financial statements should be read in conjunction with the audited financial statements and notes thereto included in the Company's annual report on Form 10-K for the year ended December&amp;nbsp;31, 2009. The preparation of financial statements in conformity with GAAP requires management to make estimates and assumptions that affect the reported amounts of assets and liabilities and disclosure of contingent assets and liabilities at the date of the financial statements and the reported amounts of revenues and expenses during the reporting period. Actual results could differ from those estimates. Significant estimates affecting the accompanying financial statements include the valuation of mortgage-backed securities ("MBS") and derivative instruments. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 12px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;Since inception, the Company had limited its exposure to credit losses on its portfolio of securities by purchasing MBS guaranteed by Fannie Mae and Freddie Mac. The portfolio is diversified to avoid undue exposure to loan originator, geographic and other types of concentration. The Company manages the risk of prepayments of the underlying mortgages by creating a diversified portfolio with a variety of prepayment characteristics. See footnote 4 for additional information on MBS. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 12px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;The Company is engaged in various trading and brokerage activities including derivative interest rate swap agreements in which counterparties primarily include broker-dealers, banks, and other financial institutions. In the event counterparties do not fulfill their obligations, the Company may be exposed to risk of loss. The risk of default depends on the creditworthiness of the counterparty and/or issuer of the instrument. It is the Company's policy to review, as necessary, the credit standing for each counterparty. See Note 7 for additional information on interest rate swap agreements. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 0px; margin-bottom: 0px;"&gt;&lt;font class="_mt" size="1"&gt; &lt;/font&gt;&amp;nbsp;&lt;/p&gt;
&lt;p style="margin-top: 18px; margin-bottom: 0px; margin-left: 2%;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;&lt;i&gt;Financial Instruments &lt;/i&gt;&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;The Company considers its cash and cash equivalents, restricted cash, MBS (settled and unsettled), forward purchase commitments, debt security held to maturity, accrued interest receivable, accounts payable, derivative instruments, repurchase agreements and accrued interest payable to meet the definition of financial instruments. The carrying amount of cash and cash equivalents, restricted cash, accrued interest receivable and accounts payable approximated their fair value due to the short maturities of these instruments. See Note 4 for discussion of the fair value of MBS securities and forward purchase commitments. See Note 5 for discussion of the fair value of the held to maturity debt security. See Note 7 for discussion of the fair value of derivatives. The carrying amount of the repurchase agreements and accrued interest payable is deemed to approximate fair value since the lines are based upon a variable rate of interest. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 18px; margin-bottom: 0px; margin-left: 2%;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;&lt;i&gt;Mortgage-Backed Securities &lt;/i&gt;&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;The Company invests in Agency MBS representing interests in or obligations backed by pools of single-family adjustable-rate mortgage loans. Guidance under the Financial Accounting Standards Board ("FASB") Accounting Standards Codification ("ASC") Topic on Investments requires the Company to classify its investments as either trading, available-for-sale or held-to-maturity securities. Management determines the appropriate classifications of the securities at the time they are acquired and evaluates the appropriateness of such classifications at each balance sheet date. The Company currently classifies all of its MBS as available-for-sale. All assets that are classified as available-for-sale are carried at fair value and unrealized gains and losses are included in other comprehensive income or loss as a component of shareholders' equity. The estimated fair values of MBS are determined by management by obtaining valuations for its MBS from three separate and independent sources and averaging these valuations. Security purchase and sale transactions are recorded on the trade date. Gains or losses realized from the sale of securities are included in income and are determined using the specific identification method. Firm purchase commitments to acquire "when issued" or to-be-announced securities ("TBA" securities) are recorded at fair value in accordance with ASC Topic 815, &lt;i&gt;Derivatives and Hedging. &lt;/i&gt;The fair value of these purchase commitments is included in other assets or liabilities in the accompanying balance sheets. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 12px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;The Company assesses its investment securities for other-than-temporary impairment on at least a quarterly basis.&amp;nbsp;When the fair value of an investment is less than its amortized cost at the balance sheet date of the reporting period for which impairment is assessed, the impairment is designated as either "temporary" or "other-than-temporary."&amp;nbsp;In deciding on whether or not a security is other than temporarily impaired, the Company uses a two step evaluation process. First, the Company determines whether it has made any decision to sell a security that is in an unrealized loss position, or, if not the Company determines whether it is more likely than not that the Company will be required to sell the security prior to recovering its amortized cost basis. If the answer to either of these questions is "yes" then the security is considered other-than-temporarily impaired. There were no such impairment losses recognized during the periods presented. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 18px; margin-bottom: 0px; margin-left: 2%;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;&lt;i&gt;Interest Income &lt;/i&gt;&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;Interest income is earned and recognized based on the outstanding principal amount of the investment securities and their contractual terms. Premiums and discounts associated with the purchase of the investment securities are amortized or accreted into interest income over the actual lives of the securities using the effective interest method. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 18px; margin-bottom: 0px; margin-left: 2%;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;&lt;i&gt;Income Taxes &lt;/i&gt;&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;The Company has elected to be taxed as a REIT under the Code. The Company will generally not be subject to Federal income tax to the extent that it distributes 100% of its taxable income, after application of available tax attributes, within the time limits prescribed by the Code and as long as it satisfies the ongoing REIT requirements including meeting certain asset, income and stock ownership tests. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 0px; margin-bottom: 0px;"&gt;&lt;font class="_mt" size="1"&gt; &lt;/font&gt;&amp;nbsp;&lt;/p&gt;
&lt;p style="margin-top: 0px; margin-bottom: 0px; margin-left: 2%;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;&lt;i&gt;Share-Based Compensation &lt;/i&gt;&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;Share-based compensation is accounted for under the guidance included in the ASC Topic on Stock Compensation. For share and share-based awards issued to employees, a compensation charge is recorded in earnings based on the fair value of the award. For transactions with non-employees in which services are performed in exchange for the Company's common stock or other equity instruments, the transactions are recorded on the basis of the fair value of the service received or the fair value of the equity instruments issued, whichever is more readily measurable at the date of issuance. The Company's share-based compensation transactions resulted in compensation expense of $347 and $321 for the three months ended June&amp;nbsp;30, 2010 and 2009, respectively. The Company's share-based compensation transactions resulted in compensation expense of $690 and $632 for the six months ended June&amp;nbsp;30, 2010 and 2009, respectively. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 18px; margin-bottom: 0px; margin-left: 2%;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;&lt;i&gt;Earnings Per Common Share (EPS) &lt;/i&gt;&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;Basic EPS is computed by dividing net income available to holders of common stock by the weighted average number of shares of common stock outstanding during the period. Diluted EPS is computed by dividing net income available to holders of common stock by the weighted average shares of common stock and common equivalent shares outstanding during the period. For the diluted EPS calculation, common equivalent shares outstanding includes the weighted average number of shares of common stock outstanding adjusted for the effect of dilutive unexercised stock options. &lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 18px; margin-bottom: 0px; margin-left: 2%;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;&lt;b&gt;&lt;i&gt;Recent Accounting Pronouncements &lt;/i&gt;&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
&lt;p style="margin-top: 6px; margin-bottom: 0px;"&gt;&lt;font style="font-family: Times New Roman;" class="_mt" size="2"&gt;In January 2010, the FASB issued an Accounting Standards Update ("ASU") on &lt;i&gt;Improving Disclosures About Fair Value Measurements&lt;/i&gt;, which clarifies certain existing disclosure requirements and requires additional disclosures for recurring and nonrecurring fair value measurements. These additional disclosures include amounts and reasons for significant transfers between Level 1 and Level 2 of the fair value hierarchy; significant transfers in and out of Level 3 of the fair value hierarchy; and information about purchases, sales, issuances and settlements on a gross basis in the reconciliation of recurring Level 3 measurements. The requirements of this standard are effective for periods beginning after December&amp;nbsp;15, 2009, with the exception of the requirement of information about purchases, sales, issuances and settlements of Level 3 measurements, which becomes effective for periods ending after December&amp;nbsp;15, 2010. The Company adopted the guidance related to Level 1 and Level 2 disclosures and the adoption did not have a material effect on the Company's financial statements. &lt;/font&gt;&lt;/p&gt; &lt;/div&gt;</NonNumbericText>
          <NonNumericTextHeader>2. Summary of Significant Accounting Policies
Basis of Presentation and Use of Estimates
The accompanying unaudited financial statements have been prepared</NonNumericTextHeader>
          <FootnoteIndexer />
          <hasSegments>false</hasSegments>
          <hasScenarios>false</hasScenarios>
          <DisplayDateInUSFormat>false</DisplayDateInUSFormat>
        </Cell>
      </Cells>
      <OriginalInstanceReportColumns />
      <ElementDataType>us-types:textBlockItemType</ElementDataType>
      <SimpleDataType>textblock</SimpleDataType>
      <ElementDefenition>This element may be used to describe all significant accounting policies of the reporting entity.</ElementDefenition>
      <ElementReferences>Reference 1: http://www.xbrl.org/2003/role/presentationRef
 -Publisher AICPA
 -Name Accounting Principles Board Opinion (APB)
 -Number 22
 -Paragraph 8

</ElementReferences>
      <IsTotalLabel>false</IsTotalLabel>
    </Row>
  </Rows>
  <Footnotes />
  <NumberOfCols>1</NumberOfCols>
  <NumberOfRows>1</NumberOfRows>
  <HasScenarios>false</HasScenarios>
  <MonetaryRoundingLevel>UnKnown</MonetaryRoundingLevel>
  <SharesRoundingLevel>UnKnown</SharesRoundingLevel>
  <PerShareRoundingLevel>UnKnown</PerShareRoundingLevel>
  <HasPureData>false</HasPureData>
  <SharesShouldBeRounded>true</SharesShouldBeRounded>
</InstanceReport>
