<?xml version="1.0" encoding="us-ascii"?><InstanceReport xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><Version>2.4.0.8</Version><ReportLongName>4070 - Disclosure - Secured Borrowings and Collateral (Details)</ReportLongName><DisplayLabelColumn>true</DisplayLabelColumn><ShowElementNames>false</ShowElementNames><RoundingOption>In Millions, unless otherwise specified</RoundingOption><HasEmbeddedReports>false</HasEmbeddedReports><Columns><Column FlagID="0"><Id>1</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>false</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>I2013Q2_ResidentialMortgageMember_1</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0001542299</EntityValue><PeriodDisplayName /><PeriodType>instant</PeriodType><PeriodStartDate>2013-06-30T00:00:00</PeriodStartDate><PeriodEndDate>0001-01-01T00:00:00</PeriodEndDate><Segments /><Scenarios /></contextRef><UPS><UnitProperty><UnitID>USD</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><MultiplyMeasures><Measure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></Measure><Measure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></Measure></MultiplyMeasures><NumeratorMeasure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></NumeratorMeasure><DenominatorMeasure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></DenominatorMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="Calendar" Id="0" Label="Jun. 30, 2013" /></Labels></Column></Columns><Rows><Row FlagID="0"><Id>1</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>0</Level><ElementName /><ElementPrefix /><IsBaseElement>true</IsBaseElement><BalanceType /><PeriodType /><IsReportTitle>false</IsReportTitle><IsSegmentTitle>true</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole /><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><OriginalInstanceReportColumn FlagID="0"><Id>1</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>true</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>I2013Q2_ResidentialMortgageMember_1</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0001542299</EntityValue><PeriodDisplayName /><PeriodType>instant</PeriodType><PeriodStartDate>2013-06-30T00:00:00</PeriodStartDate><PeriodEndDate>0001-01-01T00:00:00</PeriodEndDate><Segments><anyType xsi:type="Segment"><IsDefaultForEntity>false</IsDefaultForEntity><Name /><IsFromEntityGroup>false</IsFromEntityGroup><ValueName>First mortgage loans on owner-occupied residential property</ValueName><ValueType>us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAxis</ValueType><Namespace>xbrldi</Namespace><Schema>http://xbrl.org/2006/xbrldi</Schema><DimensionInfo><Id>us-gaap_ResidentialMortgageMember</Id><dimensionId>us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAxis</dimensionId><type>explicitMember</type></DimensionInfo></anyType></Segments><Scenarios /></contextRef><UPS><UnitProperty><UnitID>USD</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><MultiplyMeasures><Measure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></Measure><Measure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></Measure></MultiplyMeasures><NumeratorMeasure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></NumeratorMeasure><DenominatorMeasure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></DenominatorMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAxis:us-gaap_ResidentialMortgageMember" Id="0" Label="First mortgage loans on owner-occupied residential property" /><Label Key="Calendar" Id="1" Label="6/30/2013" /></Labels></OriginalInstanceReportColumn><ElementDataType>na</ElementDataType><SimpleDataType>na</SimpleDataType><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>First mortgage loans on owner-occupied residential property</Label></Row><Row FlagID="0"><Id>2</Id><IsAbstractGroupTitle>true</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>3</Level><ElementName>us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralLineItems</ElementName><ElementPrefix>us-gaap_</ElementPrefix><IsBaseElement>true</IsBaseElement><BalanceType>na</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>terseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:stringItemType</ElementDataType><SimpleDataType>string</SimpleDataType><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>Secured borrowings and collateral</Label></Row><Row FlagID="0"><Id>3</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>4</Level><ElementName>us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForFederalHomeLoanBank</ElementName><ElementPrefix>us-gaap_</ElementPrefix><IsBaseElement>true</IsBaseElement><BalanceType>debit</BalanceType><PeriodType>instant</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>terseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="I2013Q2_ResidentialMortgageMember_1" UnitID="USD"><Id>1</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>86100000</NumericAmount><RoundedNumericAmount>86.1</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:monetaryItemType</ElementDataType><SimpleDataType>monetary</SimpleDataType><ElementDefenition>The carrying amount as of the date of the latest financial statement presented of securities which are owned but transferred to serve as collateral for FHLB debt obligations, and for which the transferee does not have the right by contract or custom to sell or re-pledge them to an unrelated party.</ElementDefenition><ElementReferences>Reference 1: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Topic 860

 -SubTopic 30

 -Section 50

 -Paragraph 1A

 -Subparagraph (b)(1)(i)

 -URI http://asc.fasb.org/extlink&amp;oid=7523297&amp;loc=SL6224234-111729



Reference 2: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name FASB Staff Position (FSP)

 -Number FAS140-4/FIN46(R)-8

 -Paragraph B6

 -Subparagraph b

 -LegacyDoc This reference is SUPERSEDED by the Accounting Standards Codification effective for interim and annual periods ending after September 15, 2009.  This reference is included to help users transition from the previous accounting hierarchy and will be removed from future versions of this taxonomy.



Reference 3: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Statement of Financial Accounting Standard (FAS)

 -Number 140

 -Paragraph 17

 -Subparagraph a(2)

 -LegacyDoc This reference is SUPERSEDED by the Accounting Standards Codification effective for interim and annual periods ending after September 15, 2009.  This reference is included to help users transition from the previous accounting hierarchy and will be removed from future versions of this taxonomy.



</ElementReferences><IsTotalLabel>false</IsTotalLabel><UnitID>2</UnitID><Label>Amount of qualified collateral pledged on FHLB Boston advances</Label></Row><Row FlagID="0"><Id>4</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>0</Level><ElementName /><ElementPrefix /><IsBaseElement>true</IsBaseElement><BalanceType /><PeriodType /><IsReportTitle>false</IsReportTitle><IsSegmentTitle>true</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole /><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><OriginalInstanceReportColumn FlagID="0"><Id>2</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>true</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>I2013Q2_MortgageBackedSecuritiesMember</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0001542299</EntityValue><PeriodDisplayName /><PeriodType>instant</PeriodType><PeriodStartDate>2013-06-30T00:00:00</PeriodStartDate><PeriodEndDate>0001-01-01T00:00:00</PeriodEndDate><Segments><anyType xsi:type="Segment"><IsDefaultForEntity>false</IsDefaultForEntity><Name /><IsFromEntityGroup>false</IsFromEntityGroup><ValueName>Mortgage-backed securities</ValueName><ValueType>us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAxis</ValueType><Namespace>xbrldi</Namespace><Schema>http://xbrl.org/2006/xbrldi</Schema><DimensionInfo><Id>us-gaap_MortgageBackedSecuritiesMember</Id><dimensionId>us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAxis</dimensionId><type>explicitMember</type></DimensionInfo></anyType></Segments><Scenarios /></contextRef><UPS><UnitProperty><UnitID>USD</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><MultiplyMeasures><Measure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></Measure><Measure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></Measure></MultiplyMeasures><NumeratorMeasure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></NumeratorMeasure><DenominatorMeasure><MeasureSchema /><MeasureValue /><MeasureNamespace /><MeasureDescription /><MeasureType /></DenominatorMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAxis:us-gaap_MortgageBackedSecuritiesMember" Id="0" Label="Mortgage-backed securities" /><Label Key="Calendar" Id="1" Label="6/30/2013" /></Labels></OriginalInstanceReportColumn><ElementDataType>na</ElementDataType><SimpleDataType>na</SimpleDataType><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>Mortgage-backed securities</Label></Row><Row FlagID="0"><Id>5</Id><IsAbstractGroupTitle>true</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>3</Level><ElementName>us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralLineItems</ElementName><ElementPrefix>us-gaap_</ElementPrefix><IsBaseElement>true</IsBaseElement><BalanceType>na</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>terseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:stringItemType</ElementDataType><SimpleDataType>string</SimpleDataType><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>Secured borrowings and collateral</Label></Row><Row FlagID="0"><Id>6</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>4</Level><ElementName>us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForFederalHomeLoanBank</ElementName><ElementPrefix>us-gaap_</ElementPrefix><IsBaseElement>true</IsBaseElement><BalanceType>debit</BalanceType><PeriodType>instant</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>terseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="I2013Q2_MortgageBackedSecuritiesMember" UnitID="USD"><Id>1</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>13600000</NumericAmount><RoundedNumericAmount>13.6</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:monetaryItemType</ElementDataType><SimpleDataType>monetary</SimpleDataType><ElementDefenition>The carrying amount as of the date of the latest financial statement presented of securities which are owned but transferred to serve as collateral for FHLB debt obligations, and for which the transferee does not have the right by contract or custom to sell or re-pledge them to an unrelated party.</ElementDefenition><ElementReferences>Reference 1: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Topic 860

 -SubTopic 30

 -Section 50

 -Paragraph 1A

 -Subparagraph (b)(1)(i)

 -URI http://asc.fasb.org/extlink&amp;oid=7523297&amp;loc=SL6224234-111729



Reference 2: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name FASB Staff Position (FSP)

 -Number FAS140-4/FIN46(R)-8

 -Paragraph B6

 -Subparagraph b

 -LegacyDoc This reference is SUPERSEDED by the Accounting Standards Codification effective for interim and annual periods ending after September 15, 2009.  This reference is included to help users transition from the previous accounting hierarchy and will be removed from future versions of this taxonomy.



Reference 3: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Statement of Financial Accounting Standard (FAS)

 -Number 140

 -Paragraph 17

 -Subparagraph a(2)

 -LegacyDoc This reference is SUPERSEDED by the Accounting Standards Codification effective for interim and annual periods ending after September 15, 2009.  This reference is included to help users transition from the previous accounting hierarchy and will be removed from future versions of this taxonomy.



</ElementReferences><IsTotalLabel>false</IsTotalLabel><UnitID>2</UnitID><Label>Amount of qualified collateral pledged on FHLB Boston advances</Label></Row></Rows><Footnotes /><IsEquityReport>false</IsEquityReport><ReportName>Secured Borrowings and Collateral (Details) (USD $)</ReportName><MonetaryRoundingLevel>HundredThousands</MonetaryRoundingLevel><SharesRoundingLevel>UnKnown</SharesRoundingLevel><PerShareRoundingLevel>UnKnown</PerShareRoundingLevel><ExchangeRateRoundingLevel>UnKnown</ExchangeRateRoundingLevel><HasCustomUnits>true</HasCustomUnits><IsEmbedReport>false</IsEmbedReport><IsMultiCurrency>false</IsMultiCurrency><ReportType>Sheet</ReportType><RoleURI>http://www.georgetownsb.com/role/DisclosureSecuredBorrowingsAndCollateralDetails</RoleURI><NumberOfCols>1</NumberOfCols><NumberOfRows>6</NumberOfRows></InstanceReport>
