<?xml version="1.0" encoding="us-ascii"?><InstanceReport xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><Version>2.4.0.8</Version><ReportLongName>2419401 - Disclosure - Accounting for Income Taxes (Details)</ReportLongName><DisplayLabelColumn>true</DisplayLabelColumn><ShowElementNames>false</ShowElementNames><RoundingOption>In Millions, unless otherwise specified</RoundingOption><HasEmbeddedReports>false</HasEmbeddedReports><Columns><Column FlagID="0"><Id>1</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>true</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>I2011Q4_us-gaap_IncomeTaxAuthorityAxis_us-gaap_InternalRevenueServiceIRSMember</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0000880460</EntityValue><PeriodDisplayName /><PeriodType>instant</PeriodType><PeriodStartDate>2012-01-28T00:00:00</PeriodStartDate><PeriodEndDate>0001-01-01T00:00:00</PeriodEndDate><Segments><anyType xsi:type="Segment"><IsDefaultForEntity>false</IsDefaultForEntity><Name /><IsFromEntityGroup>false</IsFromEntityGroup><ValueName>Internal Revenue Service (IRS) [Member]</ValueName><ValueType>us-gaap_IncomeTaxAuthorityAxis</ValueType><Namespace>xbrldi</Namespace><Schema>http://xbrl.org/2006/xbrldi</Schema><DimensionInfo><Id>us-gaap_InternalRevenueServiceIRSMember</Id><dimensionId>us-gaap_IncomeTaxAuthorityAxis</dimensionId><type>explicitMember</type></DimensionInfo></anyType></Segments><Scenarios /></contextRef><UPS><UnitProperty><UnitID>usd</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="Calendar" Id="0" Label="Jan. 28, 2012" /><Label Key="us-gaap_IncomeTaxAuthorityAxis:us-gaap_InternalRevenueServiceIRSMember" Id="1" Label="Internal Revenue Service (IRS) [Member]" /></Labels></Column><Column FlagID="0"><Id>2</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode>USD</CurrencyCode><FootnoteIndexer /><hasSegments>true</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol>$</CurrencySymbol><contextRef><ContextID>D2010Q3QTD_perf_IncomeTaxBySourceAxis_perf_OperatingLossCarrybackMember</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0000880460</EntityValue><PeriodDisplayName /><PeriodType>duration</PeriodType><PeriodStartDate>2010-08-01T00:00:00</PeriodStartDate><PeriodEndDate>2010-10-30T00:00:00</PeriodEndDate><Segments><anyType xsi:type="Segment"><IsDefaultForEntity>false</IsDefaultForEntity><Name /><IsFromEntityGroup>false</IsFromEntityGroup><ValueName>Operating Loss Carryback [Member]</ValueName><ValueType>perf_IncomeTaxBySourceAxis</ValueType><Namespace>xbrldi</Namespace><Schema>http://xbrl.org/2006/xbrldi</Schema><DimensionInfo><Id>perf_OperatingLossCarrybackMember</Id><dimensionId>perf_IncomeTaxBySourceAxis</dimensionId><type>explicitMember</type></DimensionInfo></anyType></Segments><Scenarios /></contextRef><UPS><UnitProperty><UnitID>usd</UnitID><UnitType>Standard</UnitType><StandardMeasure><MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema><MeasureValue>USD</MeasureValue><MeasureNamespace>iso4217</MeasureNamespace><MeasureDescription /><MeasureType /></StandardMeasure><Scale>0</Scale></UnitProperty></UPS><CurrencyCode>USD</CurrencyCode><OriginalCurrencyCode>USD</OriginalCurrencyCode></MCU><CurrencySymbol>$</CurrencySymbol><Labels><Label Key="CalendarSupplement" Id="0" Label="3 Months Ended" /><Label Key="Calendar" Id="1" Label="Oct. 30, 2010" /><Label Key="perf_IncomeTaxBySourceAxis:perf_OperatingLossCarrybackMember" Id="2" Label="Operating Loss Carryback [Member]" /></Labels></Column></Columns><Rows><Row FlagID="0"><Id>1</Id><IsAbstractGroupTitle>true</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>3</Level><ElementName>perf_IncomeTaxesLineItems</ElementName><ElementPrefix>perf_</ElementPrefix><IsBaseElement>false</IsBaseElement><BalanceType>na</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>terseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="" UnitID=""><Id>2</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:stringItemType</ElementDataType><SimpleDataType>string</SimpleDataType><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>Income Taxes [Line Items]</Label></Row><Row FlagID="0"><Id>2</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>4</Level><ElementName>us-gaap_IncomeTaxExpenseBenefit</ElementName><ElementPrefix>us-gaap_</ElementPrefix><IsBaseElement>true</IsBaseElement><BalanceType>debit</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>true</IsReverseSign><PreferredLabelRole>negatedTerseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="D2010Q3QTD_perf_IncomeTaxBySourceAxis_perf_OperatingLossCarrybackMember" UnitID="usd"><Id>2</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>2500000</NumericAmount><RoundedNumericAmount>2.5</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:monetaryItemType</ElementDataType><SimpleDataType>monetary</SimpleDataType><ElementDefenition>Amount of current income tax expense (benefit) and deferred income tax expense (benefit) pertaining to continuing operations.</ElementDefenition><ElementReferences>Reference 1: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Topic 235

 -SubTopic 10

 -Section S99

 -Paragraph 1

 -Subparagraph (SX 210.4-08.(h))

 -URI http://asc.fasb.org/extlink&amp;oid=26873400&amp;loc=d3e23780-122690



Reference 2: http://www.xbrl.org/2003/role/presentationRef

 -Publisher SEC

 -Name Regulation S-X (SX)

 -Number 210

 -Section 08

 -Paragraph h

 -Article 4



Reference 3: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Glossary Income Tax Expense (or Benefit)

 -URI http://asc.fasb.org/extlink&amp;oid=6515339



Reference 4: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Topic 740

 -SubTopic 10

 -Section 50

 -Paragraph 9

 -Subparagraph (a),(b)

 -URI http://asc.fasb.org/extlink&amp;oid=6907707&amp;loc=d3e32639-109319



</ElementReferences><IsTotalLabel>false</IsTotalLabel><UnitID>2</UnitID><Label>Income Tax Expense (Benefit)</Label></Row><Row FlagID="0"><Id>3</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>4</Level><ElementName>us-gaap_IncomeTaxExaminationLiabilityRefundAdjustmentFromSettlementWithTaxingAuthority</ElementName><ElementPrefix>us-gaap_</ElementPrefix><IsBaseElement>true</IsBaseElement><BalanceType>credit</BalanceType><PeriodType>instant</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>true</IsReverseSign><PreferredLabelRole>negatedTerseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="I2011Q4_us-gaap_IncomeTaxAuthorityAxis_us-gaap_InternalRevenueServiceIRSMember" UnitID="usd"><Id>1</Id><IsNumeric>true</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>2400000</NumericAmount><RoundedNumericAmount>2.4</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode>USD</CurrencyCode><CurrencySymbol>$</CurrencySymbol><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>true</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell><Cell FlagID="0" ContextID="" UnitID=""><Id>2</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:monetaryItemType</ElementDataType><SimpleDataType>monetary</SimpleDataType><ElementDefenition>The amount of the additional liability or refund received or expected based on a final settlement with a taxing authority.</ElementDefenition><ElementReferences>Reference 1: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Topic 740

 -SubTopic 10

 -Section 50

 -Paragraph 15

 -Subparagraph (c)

 -URI http://asc.fasb.org/extlink&amp;oid=6907707&amp;loc=d3e32718-109319



</ElementReferences><IsTotalLabel>false</IsTotalLabel><UnitID>2</UnitID><Label>Income Tax Examination, Liability (Refund) Adjustment from Settlement with Taxing Authority</Label></Row></Rows><Footnotes /><IsEquityReport>false</IsEquityReport><ReportName>Accounting for Income Taxes (Details) (USD $)</ReportName><MonetaryRoundingLevel>HundredThousands</MonetaryRoundingLevel><SharesRoundingLevel>UnKnown</SharesRoundingLevel><PerShareRoundingLevel>UnKnown</PerShareRoundingLevel><ExchangeRateRoundingLevel>UnKnown</ExchangeRateRoundingLevel><HasCustomUnits>true</HasCustomUnits><IsEmbedReport>false</IsEmbedReport><IsMultiCurrency>false</IsMultiCurrency><ReportType>Sheet</ReportType><RoleURI>http://www.perfumania.com/role/AccountingForIncomeTaxesDetails</RoleURI><NumberOfCols>2</NumberOfCols><NumberOfRows>3</NumberOfRows></InstanceReport>
