<?xml version="1.0" encoding="us-ascii"?><InstanceReport xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><Version>2.4.0.8</Version><ReportLongName>050 - Disclosure - Note 5 - Loans (Details) - Summary of interest foregone on non-accrual loans and loans classified as TDR:</ReportLongName><DisplayLabelColumn>true</DisplayLabelColumn><ShowElementNames>false</ShowElementNames><RoundingOption>In Thousands, unless otherwise specified</RoundingOption><HasEmbeddedReports>false</HasEmbeddedReports><Columns><Column FlagID="0"><Id>1</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>false</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>c20_From1Apr2013To30Jun2013</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0000923139</EntityValue><PeriodDisplayName /><PeriodType>duration</PeriodType><PeriodStartDate>2013-04-01T00:00:00</PeriodStartDate><PeriodEndDate>2013-06-30T00:00:00</PeriodEndDate><Segments /><Scenarios /></contextRef><UPS><UnitProperty><UnitID>usd</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="CalendarSupplement" Id="0" Label="3 Months Ended" /><Label Key="Calendar" Id="1" Label="Jun. 30, 2013" /></Labels></Column><Column FlagID="0"><Id>2</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>false</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>c21_From1Apr2012To30Jun2012</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0000923139</EntityValue><PeriodDisplayName /><PeriodType>duration</PeriodType><PeriodStartDate>2012-04-01T00:00:00</PeriodStartDate><PeriodEndDate>2012-06-30T00:00:00</PeriodEndDate><Segments /><Scenarios /></contextRef><UPS><UnitProperty><UnitID>usd</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="CalendarSupplement" Id="0" Label="3 Months Ended" /><Label Key="Calendar" Id="1" Label="Jun. 30, 2012" /></Labels></Column><Column FlagID="0"><Id>3</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>false</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>c22_From1Jan2013To30Jun2013</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0000923139</EntityValue><PeriodDisplayName /><PeriodType>duration</PeriodType><PeriodStartDate>2013-01-01T00:00:00</PeriodStartDate><PeriodEndDate>2013-06-30T00:00:00</PeriodEndDate><Segments /><Scenarios /></contextRef><UPS><UnitProperty><UnitID>usd</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="CalendarSupplement" Id="0" Label="6 Months Ended" /><Label Key="Calendar" Id="1" Label="Jun. 30, 2013" /></Labels></Column><Column FlagID="0"><Id>4</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>false</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>c23_From1Jan2012To30Jun2012</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0000923139</EntityValue><PeriodDisplayName /><PeriodType>duration</PeriodType><PeriodStartDate>2012-01-01T00:00:00</PeriodStartDate><PeriodEndDate>2012-06-30T00:00:00</PeriodEndDate><Segments /><Scenarios /></contextRef><UPS><UnitProperty><UnitID>usd</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="CalendarSupplement" Id="0" Label="6 Months Ended" /><Label Key="Calendar" Id="1" Label="Jun. 30, 2012" /></Labels></Column></Columns><Rows><Row FlagID="0"><Id>1</Id><IsAbstractGroupTitle>true</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>1</Level><ElementName>ffic_SummaryOfInterestForegoneOnNonAccrualLoansAndLoansClassifiedAsTdrAbstract</ElementName><ElementPrefix>ffic_</ElementPrefix><IsBaseElement>false</IsBaseElement><BalanceType>na</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="" UnitID=""><Id>2</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="" UnitID=""><Id>3</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="" UnitID=""><Id>4</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:stringItemType</ElementDataType><SimpleDataType>string</SimpleDataType><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>Summary of interest foregone on non-accrual loans and loans classified as TDR: [Abstract]</Label></Row><Row FlagID="0"><Id>2</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>2</Level><ElementName>ffic_LoansAndLeasesReceivableImpairedTroubledDebtAndNonAccrualLoansInterestIncome</ElementName><ElementPrefix>ffic_</ElementPrefix><IsBaseElement>false</IsBaseElement><BalanceType>credit</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>label</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="c20_From1Apr2013To30Jun2013" UnitID="usd"><Id>1</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>1697000</NumericAmount><RoundedNumericAmount>1697</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c21_From1Apr2012To30Jun2012" UnitID="usd"><Id>2</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>2295000</NumericAmount><RoundedNumericAmount>2295</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c22_From1Jan2013To30Jun2013" UnitID="usd"><Id>3</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>3510000</NumericAmount><RoundedNumericAmount>3510</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c23_From1Jan2012To30Jun2012" UnitID="usd"><Id>4</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>4733000</NumericAmount><RoundedNumericAmount>4733</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:monetaryItemType</ElementDataType><SimpleDataType>monetary</SimpleDataType><ElementDefenition>The gross interest income that would have been recorded in the period on troubled debt restructurings and nonaccrual loans, if the loans had been current in accordance with their original terms and had been outstanding throughout the period or since origination, if held for part of the period.</ElementDefenition><ElementReferences>No definition available.</ElementReferences><IsTotalLabel>false</IsTotalLabel><UnitID>2</UnitID><Label>Interest income that would have been recognized had the loans performed in accordance with their original terms</Label></Row><Row FlagID="0"><Id>3</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>2</Level><ElementName>ffic_LoansAndLeasesReceivableTroubledDebtAndNonAccrualLoansInterestIncomeRecognized</ElementName><ElementPrefix>ffic_</ElementPrefix><IsBaseElement>false</IsBaseElement><BalanceType>credit</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>true</IsReverseSign><PreferredLabelRole>negatedLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="c20_From1Apr2013To30Jun2013" UnitID="usd"><Id>1</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>220000</NumericAmount><RoundedNumericAmount>220</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c21_From1Apr2012To30Jun2012" UnitID="usd"><Id>2</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>218000</NumericAmount><RoundedNumericAmount>218</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c22_From1Jan2013To30Jun2013" UnitID="usd"><Id>3</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>496000</NumericAmount><RoundedNumericAmount>496</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c23_From1Jan2012To30Jun2012" UnitID="usd"><Id>4</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>492000</NumericAmount><RoundedNumericAmount>492</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:monetaryItemType</ElementDataType><SimpleDataType>monetary</SimpleDataType><ElementDefenition>Interest income from troubled debt and non-accrual loans which was recognized in the results of operations.</ElementDefenition><ElementReferences>No definition available.</ElementReferences><IsTotalLabel>false</IsTotalLabel><UnitID>2</UnitID><Label>Less: Interest income included in the results of operations</Label></Row><Row FlagID="0"><Id>4</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>3</Level><ElementName>ffic_LoansAndLeasesReceivableTroubledDebtAndNonAccrualLoansInterestForegone</ElementName><ElementPrefix>ffic_</ElementPrefix><IsBaseElement>false</IsBaseElement><BalanceType>credit</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>totalLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="c20_From1Apr2013To30Jun2013" UnitID="usd"><Id>1</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>1477000</NumericAmount><RoundedNumericAmount>1477</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c21_From1Apr2012To30Jun2012" UnitID="usd"><Id>2</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>2077000</NumericAmount><RoundedNumericAmount>2077</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c22_From1Jan2013To30Jun2013" UnitID="usd"><Id>3</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>3014000</NumericAmount><RoundedNumericAmount>3014</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="c23_From1Jan2012To30Jun2012" UnitID="usd"><Id>4</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>4241000</NumericAmount><RoundedNumericAmount>4241</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:monetaryItemType</ElementDataType><SimpleDataType>monetary</SimpleDataType><ElementDefenition>Interest income, relating to troubled debt and non-accrual loans, that would have been recognized had the loans performed in accordance with their original terms less the amount of interest income recognized in the results of operations.  This amount represents the total foregone interest.</ElementDefenition><ElementReferences>No definition available.</ElementReferences><IsTotalLabel>true</IsTotalLabel><UnitID>2</UnitID><Label>Total foregone interest</Label></Row></Rows><Footnotes /><IsEquityReport>false</IsEquityReport><ReportName>Note 5 - Loans (Details) - Summary of interest foregone on non-accrual loans and loans classified as TDR: (USD $)</ReportName><MonetaryRoundingLevel>Thousands</MonetaryRoundingLevel><SharesRoundingLevel>UnKnown</SharesRoundingLevel><PerShareRoundingLevel>UnKnown</PerShareRoundingLevel><ExchangeRateRoundingLevel>UnKnown</ExchangeRateRoundingLevel><HasCustomUnits>true</HasCustomUnits><IsEmbedReport>false</IsEmbedReport><IsMultiCurrency>false</IsMultiCurrency><ReportType>Sheet</ReportType><RoleURI>http://flushingbank.com/role/SummaryofinterestforegoneonnonaccrualloansandloansclassifiedasTDRTable</RoleURI><NumberOfCols>4</NumberOfCols><NumberOfRows>4</NumberOfRows></InstanceReport>
