<?xml version="1.0" encoding="us-ascii"?><InstanceReport xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><Version>2.4.0.8</Version><ReportLongName>2110100 - Disclosure - Asset Impairment and Recoveries</ReportLongName><DisplayLabelColumn>true</DisplayLabelColumn><ShowElementNames>false</ShowElementNames><RoundingOption /><HasEmbeddedReports>false</HasEmbeddedReports><Columns><Column FlagID="0"><Id>1</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode /><FootnoteIndexer /><hasSegments>false</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol /><contextRef><ContextID>D2013Q3YTD</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0000904979</EntityValue><PeriodDisplayName /><PeriodType>duration</PeriodType><PeriodStartDate>2012-11-01T00:00:00</PeriodStartDate><PeriodEndDate>2013-07-31T00:00:00</PeriodEndDate><Segments /><Scenarios /></contextRef><UPS /><CurrencyCode /><OriginalCurrencyCode /></MCU><CurrencySymbol /><Labels><Label Key="CalendarSupplement" Id="0" Label="9 Months Ended" /><Label Key="Calendar" Id="1" Label="Jul. 31, 2013" /></Labels></Column></Columns><Rows><Row FlagID="0"><Id>1</Id><IsAbstractGroupTitle>true</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>1</Level><ElementName>us-gaap_RestructuringAndRelatedActivitiesAbstract</ElementName><ElementPrefix>us-gaap_</ElementPrefix><IsBaseElement>true</IsBaseElement><BalanceType>na</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText /><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>xbrli:stringItemType</ElementDataType><SimpleDataType>string</SimpleDataType><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>Restructuring and Related Activities [Abstract]</Label></Row><Row FlagID="0"><Id>2</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>2</Level><ElementName>us-gaap_RestructuringAndRelatedActivitiesDisclosureTextBlock</ElementName><ElementPrefix>us-gaap_</ElementPrefix><IsBaseElement>true</IsBaseElement><BalanceType>na</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>terseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="D2013Q3YTD" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText>&lt;div style="font-family:Times New Roman;font-size:10pt;"&gt;&lt;div style="line-height:120%;font-size:10pt;"&gt;&lt;font style="font-family:inherit;font-size:10pt;font-weight:bold;"&gt;Asset Impairment and Recoveries&lt;/font&gt;&lt;/div&gt;&lt;div style="line-height:120%;font-size:10pt;"&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt;&lt;br clear="none"/&gt;&lt;/font&gt;&lt;/div&gt;&lt;div style="line-height:120%;text-align:justify;text-indent:48px;font-size:10pt;"&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt;Impairment recoveries of &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$117&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; were recorded during the first nine months of fiscal 2013 for cash received upon sales of assets from the Company's Mansfield Blanking facility, which was impaired in fiscal 2010.&lt;/font&gt;&lt;/div&gt;&lt;div style="line-height:120%;text-align:justify;text-indent:48px;font-size:10pt;"&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt;&lt;br clear="none"/&gt;&lt;/font&gt;&lt;/div&gt;&lt;div style="line-height:120%;text-align:justify;text-indent:48px;font-size:10pt;"&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt;During the third quarter of fiscal 2012, the Company entered into negotiations to sell its Mansfield Blanking facility, which ceased operations in December 2011. As a result, the Company recorded an asset impairment charge of &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$1,552&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; to reduce the Mansfield real property to an estimated fair value of &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$1,400&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; based on an independent assessment that considered recent sales of similar properties and a submitted offer to acquire the real property. In addition, during the third quarter of fiscal 2012, the Company recorded an impairment charge of &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$392&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; to reduce the value of long lived assets to their estimated fair value. The fair value of machinery and equipment, as determined using level &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;3&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; inputs, was &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;zero&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; as the items were worn equipment for which the Company had no further use and they have limited use and limited value in the used equipment market. Impairment recovery of &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$1,202&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; was recorded during the first nine months of fiscal 2012. The recoveries recorded in the first nine months of fiscal 2012 were for cash received upon sales of assets from the Company's Mansfield Blanking facility of &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$689&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt;, which was impaired in fiscal 2010, and the Company's Liverpool Stamping facility of &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$508&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt;, which was impaired in fiscal 2009, with the remaining &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$5&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; of recoveries coming from other assets impaired in prior periods. In addition, during the third quarter of fiscal 2012, the Company reduced a restructuring charge by &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$30&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; as a result of certain employees not meeting the requirements for obtaining severance payments associated with the restructuring charge of &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;$352&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; that the Company recorded in the third quarter of fiscal 2011, relating to a negotiated settlement with approximately &lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;color:#000000;text-decoration:none;"&gt;90&lt;/font&gt;&lt;font style="font-family:inherit;font-size:10pt;"&gt; employees for severance and health insurance related to the previously announced planned closure of the Company's plant in Mansfield, Ohio.&lt;/font&gt;&lt;/div&gt;&lt;/div&gt;</NonNumbericText><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>nonnum:textBlockItemType</ElementDataType><SimpleDataType>na</SimpleDataType><ElementDefenition>The entire disclosure for restructuring and related activities. Description of restructuring activities such as exit and disposal activities, include facts and circumstances leading to the plan, the expected plan completion date, the major types of costs associated with the plan activities, total expected costs, the accrual balance at the end of the period, and the periods over which the remaining accrual will be settled.</ElementDefenition><ElementReferences>Reference 1: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Statement of Financial Accounting Standard (FAS)

 -Number 146

 -Paragraph 20

 -LegacyDoc This reference is SUPERSEDED by the Accounting Standards Codification effective for interim and annual periods ending after September 15, 2009.  This reference is included to help users transition from the previous accounting hierarchy and will be removed from future versions of this taxonomy.



Reference 2: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Topic 420

 -SubTopic 10

 -Section S99

 -Paragraph 2

 -Subparagraph (SAB TOPIC 5.P.4)

 -URI http://asc.fasb.org/extlink&amp;oid=6394695&amp;loc=d3e140904-122747



Reference 3: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Topic 420

 -SubTopic 10

 -Section S99

 -Paragraph 1

 -Subparagraph (SAB TOPIC 5.P.3)

 -URI http://asc.fasb.org/extlink&amp;oid=6394695&amp;loc=d3e140864-122747



Reference 4: http://www.xbrl.org/2003/role/presentationRef

 -Publisher SEC

 -Name Staff Accounting Bulletin (SAB)

 -Number Topic 5

 -Section P

 -Subsection 3, 4



Reference 5: http://www.xbrl.org/2003/role/presentationRef

 -Publisher FASB

 -Name Accounting Standards Codification

 -Topic 420

 -SubTopic 10

 -Section 50

 -Paragraph 1

 -URI http://asc.fasb.org/extlink&amp;oid=6394359&amp;loc=d3e17939-110869



</ElementReferences><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>Restructuring and Related Activities Disclosure [Text Block]</Label></Row></Rows><Footnotes /><IsEquityReport>false</IsEquityReport><ReportName>Asset Impairment and Recoveries</ReportName><MonetaryRoundingLevel>UnKnown</MonetaryRoundingLevel><SharesRoundingLevel>UnKnown</SharesRoundingLevel><PerShareRoundingLevel>UnKnown</PerShareRoundingLevel><ExchangeRateRoundingLevel>UnKnown</ExchangeRateRoundingLevel><HasCustomUnits>true</HasCustomUnits><IsEmbedReport>false</IsEmbedReport><IsMultiCurrency>false</IsMultiCurrency><ReportType>Sheet</ReportType><RoleURI>http://www.shiloh.com/role/AssetImpairmentAndRecoveries</RoleURI><NumberOfCols>1</NumberOfCols><NumberOfRows>2</NumberOfRows></InstanceReport>
