﻿<?xml version="1.0" encoding="utf-8"?>
<InstanceReport xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
  <Version>2.2.0.7</Version>
  <hasSegments>false</hasSegments>
  <ReportName>Restructuring, Goodwill Impairment and Other Special Charges, Net</ReportName>
  <ReportLongName>11201 - Disclosure - Restructuring, Goodwill Impairment and Other Special Charges, Net</ReportLongName>
  <DisplayLabelColumn>true</DisplayLabelColumn>
  <ShowElementNames>false</ShowElementNames>
  <RoundingOption />
  <HasEmbeddedReports>false</HasEmbeddedReports>
  <Columns>
    <Column>
      <LabelColumn>false</LabelColumn>
      <Id>1</Id>
      <Labels>
        <Label Id="1" Label="6 Months Ended" />
        <Label Id="2" Label="Jun. 30, 2010" />
      </Labels>
      <CurrencyCode>USD</CurrencyCode>
      <FootnoteIndexer />
      <hasSegments>false</hasSegments>
      <hasScenarios>false</hasScenarios>
      <Segments />
      <Scenarios />
      <Units>
        <Unit>
          <UnitID>Unit12</UnitID>
          <UnitType>Standard</UnitType>
          <StandardMeasure>
            <MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema>
            <MeasureValue>USD</MeasureValue>
            <MeasureNamespace>iso4217</MeasureNamespace>
          </StandardMeasure>
          <Scale>0</Scale>
        </Unit>
        <Unit>
          <UnitID>Unit14</UnitID>
          <UnitType>Divide</UnitType>
          <NumeratorMeasure>
            <MeasureSchema>http://www.xbrl.org/2003/iso4217</MeasureSchema>
            <MeasureValue>USD</MeasureValue>
            <MeasureNamespace>iso4217</MeasureNamespace>
          </NumeratorMeasure>
          <DenominatorMeasure>
            <MeasureSchema>http://www.xbrl.org/2003/instance</MeasureSchema>
            <MeasureValue>shares</MeasureValue>
            <MeasureNamespace>xbrli</MeasureNamespace>
          </DenominatorMeasure>
          <Scale>0</Scale>
        </Unit>
        <Unit>
          <UnitID>Unit1</UnitID>
          <UnitType>Standard</UnitType>
          <StandardMeasure>
            <MeasureSchema>http://www.xbrl.org/2003/instance</MeasureSchema>
            <MeasureValue>shares</MeasureValue>
            <MeasureNamespace>xbrli</MeasureNamespace>
          </StandardMeasure>
          <Scale>0</Scale>
        </Unit>
      </Units>
      <CurrencySymbol>$</CurrencySymbol>
    </Column>
  </Columns>
  <Rows>
    <Row>
      <Id>5</Id>
      <Label>Restructuring, Goodwill Impairment and Other Special Charges, Net</Label>
      <Level>3</Level>
      <ElementName>us-gaap_RestructuringAndRelatedActivitiesDisclosureTextBlock</ElementName>
      <ElementPrefix>us-gaap</ElementPrefix>
      <IsBaseElement>true</IsBaseElement>
      <BalanceType>na</BalanceType>
      <PeriodType>duration</PeriodType>
      <ShortDefinition>No definition available.</ShortDefinition>
      <IsReportTitle>false</IsReportTitle>
      <IsSegmentTitle>false</IsSegmentTitle>
      <IsSubReportEnd>false</IsSubReportEnd>
      <IsCalendarTitle>false</IsCalendarTitle>
      <IsTuple>false</IsTuple>
      <IsAbstractGroupTitle>false</IsAbstractGroupTitle>
      <IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow>
      <IsEquityAdjustmentRow>false</IsEquityAdjustmentRow>
      <IsBeginningBalance>false</IsBeginningBalance>
      <IsEndingBalance>false</IsEndingBalance>
      <IsReverseSign>false</IsReverseSign>
      <PreferredLabelRole>terselabel</PreferredLabelRole>
      <IsEPS>false</IsEPS>
      <FootnoteIndexer />
      <Cells>
        <Cell>
          <Id>1</Id>
          <ShowCurrencySymbol>false</ShowCurrencySymbol>
          <IsNumeric>false</IsNumeric>
          <IsRatio>false</IsRatio>
          <DisplayZeroAsNone>false</DisplayZeroAsNone>
          <NumericAmount>0</NumericAmount>
          <RoundedNumericAmount>0</RoundedNumericAmount>
          <NonNumbericText>&lt;div&gt;
&lt;div style="font-family: 'Times New Roman',Times,serif;"&gt;
&lt;div style="margin-top: 12pt; font-size: 10pt;" align="left"&gt;&lt;b&gt;Note 12. Restructuring and Other Special Charges (Credits), Net&lt;/b&gt; &lt;/div&gt;
&lt;div style="margin-top: 6pt; font-size: 10pt;" align="left"&gt;&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;During the three and six months ended June&amp;nbsp;30, 2010, the Company recorded restructuring credits of $1&amp;nbsp;million associated with the reversal of previous restructuring reserves no longer deemed necessary. During the three and six months ended June&amp;nbsp;30, 2009, the Company recorded restructuring charges of $5&amp;nbsp;million and $22&amp;nbsp;million, respectively in connection with its previous initiative of rationalizing its cost structure in light of the decline in growth in its business units. &lt;/div&gt;
&lt;div style="margin-top: 6pt; font-size: 10pt;" align="left"&gt;&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;Restructuring costs and other special charges (credits), net, by segment are as follows: (in millions): &lt;/div&gt;
&lt;table style="font-size: 10pt; text-align: left;" cellspacing="0" cellpadding="0" width="100%" border="0"&gt;
&lt;tr valign="bottom"&gt;&lt;td width="52%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="5%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="1%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="5%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="1%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="5%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="1%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="5%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="1%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="5%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="1%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="5%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="1%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="5%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="1%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="5%"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td width="1%"&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr style="font-size: 8pt;" valign="bottom"&gt;&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td nowrap="nowrap" align="center" colspan="6"&gt;&lt;b&gt;Three Months&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td nowrap="nowrap" align="center" colspan="6"&gt;&lt;b&gt;Six Months&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr style="font-size: 8pt;" valign="bottom"&gt;&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td nowrap="nowrap" align="center" colspan="6"&gt;&lt;b&gt;Ended&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td nowrap="nowrap" align="center" colspan="6"&gt;&lt;b&gt;Ended&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr style="font-size: 8pt;" valign="bottom"&gt;&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-bottom: #000000 1px solid;" nowrap="nowrap" align="center" colspan="6"&gt;&lt;b&gt;June 30,&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-bottom: #000000 1px solid;" nowrap="nowrap" align="center" colspan="6"&gt;&lt;b&gt;June 30,&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr style="font-size: 8pt;" valign="bottom"&gt;&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-bottom: #000000 1px solid;" nowrap="nowrap" align="center" colspan="2"&gt;&lt;b&gt;2010&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-bottom: #000000 1px solid;" nowrap="nowrap" align="center" colspan="2"&gt;&lt;b&gt;2009&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-bottom: #000000 1px solid;" nowrap="nowrap" align="center" colspan="2"&gt;&lt;b&gt;2010&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-bottom: #000000 1px solid;" nowrap="nowrap" align="center" colspan="2"&gt;&lt;b&gt;2009&lt;/b&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr style="background: #cceeff;" valign="bottom"&gt;&lt;td&gt;
&lt;div style="margin-left: 15px; text-indent: -15px;"&gt;Hotel&lt;/div&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td nowrap="nowrap" align="left"&gt;$&lt;/td&gt;
&lt;td align="right"&gt;(1&lt;/td&gt;
&lt;td nowrap="nowrap"&gt;)&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td align="left"&gt;$&lt;/td&gt;
&lt;td align="right"&gt;4&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td nowrap="nowrap" align="left"&gt;$&lt;/td&gt;
&lt;td align="right"&gt;(1&lt;/td&gt;
&lt;td nowrap="nowrap"&gt;)&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td align="left"&gt;$&lt;/td&gt;
&lt;td align="right"&gt;13&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr valign="bottom"&gt;&lt;td&gt;
&lt;div style="margin-left: 15px; text-indent: -15px;"&gt;Vacation Ownership &amp;amp; Residential&lt;/div&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td align="right"&gt;&amp;#8212;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td align="right"&gt;1&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td align="right"&gt;&amp;#8212;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td align="right"&gt;9&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr style="font-size: 1px;"&gt;&lt;td&gt;
&lt;div style="margin-left: 15px; text-indent: -15px;"&gt;&amp;nbsp;&lt;/div&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-top: #000000 1px solid;" nowrap="nowrap" align="right" colspan="2"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-top: #000000 1px solid;" nowrap="nowrap" align="right" colspan="2"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-top: #000000 1px solid;" nowrap="nowrap" align="right" colspan="2"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-top: #000000 1px solid;" nowrap="nowrap" align="right" colspan="2"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr style="background: #cceeff;" valign="bottom"&gt;&lt;td&gt;
&lt;div style="margin-left: 15px; text-indent: -15px;"&gt;Total&lt;/div&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td nowrap="nowrap" align="left"&gt;$&lt;/td&gt;
&lt;td align="right"&gt;(1&lt;/td&gt;
&lt;td nowrap="nowrap"&gt;)&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td align="left"&gt;$&lt;/td&gt;
&lt;td align="right"&gt;5&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td nowrap="nowrap" align="left"&gt;$&lt;/td&gt;
&lt;td align="right"&gt;(1&lt;/td&gt;
&lt;td nowrap="nowrap"&gt;)&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td align="left"&gt;$&lt;/td&gt;
&lt;td align="right"&gt;22&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;
&lt;tr style="font-size: 1px;"&gt;&lt;td&gt;
&lt;div style="margin-left: 15px; text-indent: -15px;"&gt;&amp;nbsp;&lt;/div&gt;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-top: #000000 3px double;" nowrap="nowrap" align="right" colspan="2"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-top: #000000 3px double;" nowrap="nowrap" align="right" colspan="2"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-top: #000000 3px double;" nowrap="nowrap" align="right" colspan="2"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;
&lt;td style="border-top: #000000 3px double;" nowrap="nowrap" align="right" colspan="2"&gt;&amp;nbsp;&lt;/td&gt;
&lt;td&gt;&amp;nbsp;&lt;/td&gt;&lt;/tr&gt;&lt;/table&gt;&lt;/div&gt;
&lt;div style="margin-top: 6pt; font-size: 10pt;" align="left"&gt;&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;The Company had remaining accruals of $22&amp;nbsp;million and $26&amp;nbsp;million as of June&amp;nbsp;30, 2010 and December&amp;nbsp;31, 2009, respectively, which are primarily related to long-term liabilities for certain obligations in the vacation ownership business that are expected to be paid out in future years. &lt;/div&gt;
&lt;div&gt; &lt;/div&gt; &lt;/div&gt;</NonNumbericText>
          <NonNumericTextHeader>Note 12. Restructuring and Other Special Charges (Credits), Net
&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;During the three and six months ended June&amp;nbsp;30, 2010, the</NonNumericTextHeader>
          <FootnoteIndexer />
          <hasSegments>false</hasSegments>
          <hasScenarios>false</hasScenarios>
          <DisplayDateInUSFormat>false</DisplayDateInUSFormat>
        </Cell>
      </Cells>
      <OriginalInstanceReportColumns />
      <ElementDataType>us-types:textBlockItemType</ElementDataType>
      <SimpleDataType>textblock</SimpleDataType>
      <ElementDefenition>Description of restructuring activities including exit and disposal activities, which should include facts and circumstances leading to the plan, the expected plan completion date, the major types of costs associated with the plan activities, total expected costs, the accrual balance at the end of the period, and the periods over which the remaining accrual will be settled. This description does not include restructuring costs in connection with a business combination or discontinued operations and long-lived assets (disposal groups) sold or classified as held for sale. This element may be used as a single block of text to encapsulate the entire disclosure including data and tables.</ElementDefenition>
      <ElementReferences>Reference 1: http://www.xbrl.org/2003/role/presentationRef
 -Publisher FASB
 -Name Statement of Financial Accounting Standard (FAS)
 -Number 146
 -Paragraph 20

Reference 2: http://www.xbrl.org/2003/role/presentationRef
 -Publisher SEC
 -Name Staff Accounting Bulletin (SAB)
 -Number Topic 5
 -Section P
 -Subsection 3, 4

</ElementReferences>
      <IsTotalLabel>false</IsTotalLabel>
    </Row>
  </Rows>
  <Footnotes />
  <NumberOfCols>1</NumberOfCols>
  <NumberOfRows>1</NumberOfRows>
  <HasScenarios>false</HasScenarios>
  <MonetaryRoundingLevel>UnKnown</MonetaryRoundingLevel>
  <SharesRoundingLevel>UnKnown</SharesRoundingLevel>
  <PerShareRoundingLevel>UnKnown</PerShareRoundingLevel>
  <HasPureData>false</HasPureData>
  <SharesShouldBeRounded>true</SharesShouldBeRounded>
</InstanceReport>
